enigma-memory 0.1.18 → 0.1.22

package/apps/relay/src/federation-runtime.mjs

@@ -0,0 +1,618 @@
+import { createHash, timingSafeEqual } from 'node:crypto';
+import { EventEmitter } from 'node:events';
+
+import WebSocket, { WebSocketServer } from 'ws';
+
+import {
+  PACKET_TYPES,
+  decodePacket,
+  verifyPacketSignature,
+} from '../../../packages/mesh/src/index.js';
+
+export const FEDERATION_SERVE_PATH = '/ghostmesh/federation';
+export const FEDERATION_STORAGE_ADAPTER_SCHEMA = 'enigma.federation_storage_adapter.v1';
+
+const FEDERATION_PACKET_TYPES = new Set([
+  PACKET_TYPES.CAPABILITY_GRANT,
+  PACKET_TYPES.FEDERATED_QUERY_REQUEST,
+  PACKET_TYPES.FEDERATED_QUERY_RESPONSE,
+  PACKET_TYPES.CAPABILITY_REVOCATION,
+]);
+const PRODUCTION_MODES = new Set(['production', 'hosted', 'byoc', 'customer_byoc', 'on_prem', 'onprem']);
+const PUBLIC_KEY_RE = /^[0-9a-f]{64}$/;
+const PEER_ID_RE = /^[A-Za-z0-9._:@/-]{1,128}$/;
+const DEFAULT_MAX_FRAME_BYTES = 1024 * 1024;
+const DEFAULT_MAX_BUFFERED_BYTES = 2 * 1024 * 1024;
+const DEFAULT_MAX_QUEUED_BYTES = 2 * 1024 * 1024;
+const DEFAULT_MAX_QUEUED_FRAMES = 64;
+const DEFAULT_RECONNECT_ATTEMPTS = 3;
+const DEFAULT_RECONNECT_DELAY_MS = 100;
+const DEFAULT_CONNECT_TIMEOUT_MS = 5000;
+
+function nonEmptyString(value) {
+  return typeof value === 'string' && value.trim().length > 0;
+}
+
+function positiveSafeInteger(value, fallback, name) {
+  if (value === undefined) return fallback;
+  if (!Number.isSafeInteger(value) || value < 1) throw new TypeError(`${name} must be a positive safe integer`);
+  return value;
+}
+
+function normalizePublicKey(value, name = 'public key') {
+  const key = Buffer.isBuffer(value) || value instanceof Uint8Array
+    ? Buffer.from(value).toString('hex')
+    : String(value ?? '').toLowerCase();
+  if (!PUBLIC_KEY_RE.test(key)) throw new Error(`${name} must be a 32-byte hex value`);
+  return key;
+}
+
+function normalizePeerId(value) {
+  const peerId = String(value ?? '');
+  if (!PEER_ID_RE.test(peerId)) throw new Error('authenticated peer id is invalid');
+  return peerId;
+}
+
+function headerValue(headers, name) {
+  const value = headers?.[name.toLowerCase()];
+  return Array.isArray(value) ? value[0] : value;
+}
+
+function constantTimeStringEqual(left, right) {
+  const leftHash = createHash('sha256').update(String(left)).digest();
+  const rightHash = createHash('sha256').update(String(right)).digest();
+  return timingSafeEqual(leftHash, rightHash);
+}
+
+function validateFederationFrame(value, maxFrameBytes, expectedSenderPublicKey) {
+  const frame = Buffer.from(value);
+  if (frame.length === 0 || frame.length > maxFrameBytes) {
+    throw new Error('federation frame exceeds the authenticated binary frame limit');
+  }
+  const decoded = decodePacket(frame);
+  if (!FEDERATION_PACKET_TYPES.has(decoded.type)) throw new Error('frame is not a GhostMesh federation packet');
+  if (!verifyPacketSignature(frame)) throw new Error('federation frame signature is invalid');
+  if (expectedSenderPublicKey !== undefined &&
+      !constantTimeStringEqual(decoded.senderPubkey.toString('hex'), expectedSenderPublicKey)) {
+    throw new Error('federation frame sender is not bound to the authenticated connection identity');
+  }
+  return { frame, decoded };
+}
+
+function closeSocket(socket, code, reason) {
+  if (socket.readyState === WebSocket.OPEN || socket.readyState === WebSocket.CONNECTING) {
+    socket.close(code, String(reason).slice(0, 123));
+  }
+}
+
+function receivedBinaryFrame(data, isBinary) {
+  if (typeof isBinary === 'boolean') return isBinary;
+  return Buffer.isBuffer(data) || ArrayBuffer.isView(data) || data instanceof ArrayBuffer;
+}
+
+function sendBinary(socket, frame, maxBufferedBytes) {
+  if (socket.readyState !== WebSocket.OPEN) return Promise.resolve(false);
+  if (socket.bufferedAmount + frame.length > maxBufferedBytes) return Promise.resolve(false);
+  return new Promise((resolve) => {
+    socket.send(frame, { binary: true }, (error) => resolve(!error));
+  });
+}
+
+function storageRecord(direction, peerId, decoded, frame) {
+  return Object.freeze({
+    schema: 'enigma.federation_opaque_frame.v1',
+    direction,
+    peer_id: peerId,
+    packet_type: decoded.typeName,
+    destination: decoded.destinationHex,
+    sender_public_key: decoded.senderPubkey.toString('hex'),
+    frame: Buffer.from(frame),
+    received_at: new Date().toISOString(),
+  });
+}
+
+export function validateFederationStorageAdapter(adapter, { requireDurable = false } = {}) {
+  if (!adapter || typeof adapter !== 'object') throw new TypeError('federation storage adapter is required');
+  if (adapter.schema !== FEDERATION_STORAGE_ADAPTER_SCHEMA) {
+    throw new TypeError(`federation storage adapter schema must be ${FEDERATION_STORAGE_ADAPTER_SCHEMA}`);
+  }
+  if (!nonEmptyString(adapter.ref)) throw new TypeError('federation storage adapter ref is required');
+  if (typeof adapter.isReady !== 'function' || typeof adapter.appendOpaqueFrame !== 'function') {
+    throw new TypeError('federation storage adapter must implement isReady() and appendOpaqueFrame(record)');
+  }
+  if (requireDurable && adapter.durable !== true) {
+    throw new TypeError('production federation serve mode requires a durable external storage adapter');
+  }
+  return adapter;
+}
+
+export class MemoryFederationStorageAdapter {
+  constructor(options = {}) {
+    this.schema = FEDERATION_STORAGE_ADAPTER_SCHEMA;
+    this.ref = options.ref ?? 'memory://explicit-federation-frames';
+    this.durable = false;
+    this.maxRecords = positiveSafeInteger(options.maxRecords, 1024, 'maxRecords');
+    this.records = [];
+  }
+
+  async isReady() {
+    return true;
+  }
+
+  async appendOpaqueFrame(record) {
+    if (!Buffer.isBuffer(record?.frame)) throw new TypeError('opaque federation storage record frame must be a Buffer');
+    if (this.records.length >= this.maxRecords) this.records.shift();
+    this.records.push({ ...record, frame: Buffer.from(record.frame) });
+  }
+}
+
+export function createMemoryFederationStorageAdapter(options) {
+  return new MemoryFederationStorageAdapter(options);
+}
+
+export class FederationServeTransport extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    this.id = options.id ?? 'federation-serve-transport';
+    this.running = false;
+    this._sender = null;
+  }
+
+  async start() {
+    this.running = true;
+    return this;
+  }
+
+  async stop() {
+    this.running = false;
+  }
+
+  async send(packetBuffer, peerId) {
+    if (!this.running || typeof this._sender !== 'function') return false;
+    return await this._sender(packetBuffer, peerId);
+  }
+}
+
+export class AuthenticatedWebSocketFederationTransport extends EventEmitter {
+  constructor(options = {}) {
+    super();
+    if (!nonEmptyString(options.url)) throw new TypeError('federation WebSocket url is required');
+    this.id = options.id ?? 'authenticated-ws-federation-client';
+    this.url = options.url;
+    this.peerId = normalizePeerId(options.peerId);
+    this.publicKey = normalizePublicKey(options.publicKey, 'client public key');
+    if (!nonEmptyString(options.authorization)) throw new TypeError('client authorization is required');
+    this.authorization = options.authorization;
+    this.maxFrameBytes = positiveSafeInteger(options.maxFrameBytes, DEFAULT_MAX_FRAME_BYTES, 'maxFrameBytes');
+    this.maxBufferedBytes = positiveSafeInteger(options.maxBufferedBytes, DEFAULT_MAX_BUFFERED_BYTES, 'maxBufferedBytes');
+    this.connectTimeoutMs = positiveSafeInteger(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS, 'connectTimeoutMs');
+    this.running = false;
+    this.socket = null;
+  }
+
+  async start() {
+    if (this.running && this.socket?.readyState === WebSocket.OPEN) return this;
+    const socket = new WebSocket(this.url, {
+      headers: {
+        authorization: this.authorization,
+        'x-enigma-peer-id': this.peerId,
+        'x-enigma-public-key': this.publicKey,
+      },
+      maxPayload: this.maxFrameBytes,
+      perMessageDeflate: false,
+    });
+    this.socket = socket;
+    await new Promise((resolve, reject) => {
+      const timer = setTimeout(() => {
+        socket.terminate();
+        reject(new Error('federation WebSocket connection timed out'));
+      }, this.connectTimeoutMs);
+      timer.unref?.();
+      socket.once('open', () => {
+        clearTimeout(timer);
+        resolve();
+      });
+      socket.once('error', (error) => {
+        clearTimeout(timer);
+        reject(error);
+      });
+    });
+    this.running = true;
+    socket.on('message', (data, isBinary) => {
+      try {
+        if (!receivedBinaryFrame(data, isBinary)) throw new Error('federation WebSocket accepts binary frames only');
+        const { frame } = validateFederationFrame(data, this.maxFrameBytes);
+        this.emit('packet', { packetBuffer: frame, fromPeerId: 'federation-service', transportId: this.id });
+      } catch (error) {
+        this.emit('rejected', { error });
+        closeSocket(socket, 1008, 'invalid federation frame');
+      }
+    });
+    socket.once('close', () => {
+      this.running = false;
+      this.emit('disconnect', { peerId: 'federation-service' });
+    });
+    return this;
+  }
+
+  async stop() {
+    const socket = this.socket;
+    this.running = false;
+    this.socket = null;
+    if (!socket || socket.readyState === WebSocket.CLOSED) return;
+    await new Promise((resolve) => {
+      socket.once('close', resolve);
+      closeSocket(socket, 1000, 'transport stopped');
+      setTimeout(resolve, 250).unref?.();
+    });
+  }
+
+  async send(packetBuffer) {
+    if (!this.running || !this.socket) return false;
+    const { frame } = validateFederationFrame(packetBuffer, this.maxFrameBytes, this.publicKey);
+    return await sendBinary(this.socket, frame, this.maxBufferedBytes);
+  }
+}
+
+async function authenticateRequest(authenticator, request) {
+  const declaredPeerId = headerValue(request.headers, 'x-enigma-peer-id');
+  const declaredPublicKey = headerValue(request.headers, 'x-enigma-public-key');
+  if (!nonEmptyString(declaredPeerId) || !nonEmptyString(declaredPublicKey)) {
+    throw new Error('authenticated federation identity headers are required');
+  }
+  const result = await authenticator.authenticate({
+    authorization: headerValue(request.headers, 'authorization'),
+    peerId: declaredPeerId,
+    publicKey: declaredPublicKey,
+    remoteAddress: request.socket?.remoteAddress,
+  });
+  if (!result || result.ok === false) throw new Error('federation authentication failed');
+  const peerId = normalizePeerId(result.peerId ?? result.peer_id);
+  const publicKey = normalizePublicKey(result.publicKey ?? result.public_key, 'authenticated public key');
+  if (!constantTimeStringEqual(peerId, declaredPeerId) ||
+      !constantTimeStringEqual(publicKey, normalizePublicKey(declaredPublicKey, 'declared public key'))) {
+    throw new Error('declared federation identity does not match authenticated identity');
+  }
+  return { peerId, publicKey };
+}
+
+function rejectUpgrade(socket, statusCode, reason) {
+  if (!socket.writable) return;
+  const body = `${reason}\n`;
+  socket.end(
+    `HTTP/1.1 ${statusCode} ${statusCode === 401 ? 'Unauthorized' : 'Bad Request'}\r\n` +
+    'Connection: close\r\nContent-Type: text/plain; charset=utf-8\r\n' +
+    `Content-Length: ${Buffer.byteLength(body)}\r\n\r\n${body}`
+  );
+}
+
+export class FederationServeRuntime extends EventEmitter {
+  constructor(server, options = {}) {
+    super();
+    if (!server || typeof server.on !== 'function') throw new TypeError('HTTP server is required');
+    const mode = String(options.mode ?? 'local').toLowerCase();
+    const production = PRODUCTION_MODES.has(mode);
+    const storage = validateFederationStorageAdapter(options.storage, { requireDurable: production });
+    const authenticator = options.authenticator;
+    if (!authenticator || typeof authenticator.authenticate !== 'function') {
+      throw new TypeError('federation authenticator with authenticate() is required');
+    }
+    const authRef = options.authRef ?? authenticator.ref;
+    if (!nonEmptyString(authRef)) throw new TypeError('federation auth ref is required');
+    if (production && !nonEmptyString(options.kmsRef)) {
+      throw new TypeError('production federation serve mode requires a KMS/key-custody ref');
+    }
+
+    this.server = server;
+    this.mode = mode;
+    this.production = production;
+    this.path = options.path ?? FEDERATION_SERVE_PATH;
+    this.storage = storage;
+    this.authenticator = authenticator;
+    this.authRef = authRef;
+    this.kmsRef = options.kmsRef ?? null;
+    this.maxFrameBytes = positiveSafeInteger(options.maxFrameBytes, DEFAULT_MAX_FRAME_BYTES, 'maxFrameBytes');
+    this.maxBufferedBytes = positiveSafeInteger(options.maxBufferedBytes, DEFAULT_MAX_BUFFERED_BYTES, 'maxBufferedBytes');
+    this.maxQueuedBytes = positiveSafeInteger(options.maxQueuedBytes, DEFAULT_MAX_QUEUED_BYTES, 'maxQueuedBytes');
+    this.maxQueuedFrames = positiveSafeInteger(options.maxQueuedFrames, DEFAULT_MAX_QUEUED_FRAMES, 'maxQueuedFrames');
+    this.reconnectAttempts = positiveSafeInteger(options.reconnectAttempts, DEFAULT_RECONNECT_ATTEMPTS, 'reconnectAttempts');
+    this.reconnectDelayMs = positiveSafeInteger(options.reconnectDelayMs, DEFAULT_RECONNECT_DELAY_MS, 'reconnectDelayMs');
+    this.connectTimeoutMs = positiveSafeInteger(options.connectTimeoutMs, DEFAULT_CONNECT_TIMEOUT_MS, 'connectTimeoutMs');
+    this.upstreamUrl = options.upstreamUrl ?? null;
+    this.upstreamHeaders = options.upstreamHeaders ?? null;
+    this.expectedUpstreamPublicKeys = new Set((options.expectedUpstreamPublicKeys ?? []).map((value) => normalizePublicKey(value, 'expected upstream public key')));
+    if (production && this.upstreamUrl !== null && this.expectedUpstreamPublicKeys.size === 0) {
+      throw new TypeError('production federation forwarding requires expected upstream bridge public keys');
+    }
+    this.connections = new Map();
+    this.forwardConnections = new Set();
+    this.storageReady = false;
+    this.authReady = false;
+    this.running = false;
+    this.upstreamReady = this.upstreamUrl === null;
+    this.lastDependencyError = null;
+    this.transport = options.transport ?? new FederationServeTransport();
+    if (!(this.transport instanceof EventEmitter) || typeof this.transport.send !== 'function') {
+      throw new TypeError('federation bridge transport must be EventEmitter-compatible and implement send()');
+    }
+    this.bridge = options.bridge ?? null;
+    if (this.upstreamUrl === null && !this.bridge) throw new TypeError('terminal federation serve mode requires a GhostMesh federation bridge');
+    if (this.upstreamUrl !== null && this.bridge) throw new TypeError('federation serve mode cannot configure both an upstream and a terminal bridge');
+    if (this.bridge) {
+      if (this.bridge.started) throw new Error('federation bridge must not be started before attaching serve mode');
+      this.bridge.transport = this.transport;
+      this.bridge.manageTransport = false;
+    }
+    this.transport._sender = (packetBuffer, peerId) => this.sendToPeer(packetBuffer, peerId);
+
+    this.webSocketServer = new WebSocketServer({
+      noServer: true,
+      maxPayload: this.maxFrameBytes,
+      perMessageDeflate: false,
+      clientTracking: false,
+    });
+    this._upgradeHandler = (request, socket, head) => void this.handleUpgrade(request, socket, head);
+    server.on('upgrade', this._upgradeHandler);
+    server.once('listening', () => void this.start());
+    server.once('close', () => void this.stop());
+    if (server.listening) void this.start();
+  }
+
+  async refreshDependencies() {
+    try {
+      this.storageReady = await this.storage.isReady() === true;
+      this.authReady = typeof this.authenticator.isReady === 'function'
+        ? await this.authenticator.isReady() === true
+        : true;
+      if (!this.storageReady) throw new Error('federation storage adapter is not ready');
+      if (!this.authReady) throw new Error('federation authenticator is not ready');
+      this.lastDependencyError = null;
+    } catch (error) {
+      this.storageReady = false;
+      this.lastDependencyError = error instanceof Error ? error.message : 'federation dependency failed';
+    }
+  }
+
+  async start() {
+    if (this.running) return this;
+    await this.refreshDependencies();
+    if (!this.storageReady || !this.authReady) return this;
+    if (!this.transport.running) await this.transport.start();
+    if (this.bridge && !this.bridge.started) await this.bridge.start();
+    this.running = true;
+    if (this.upstreamUrl && this.upstreamHeaders) this.startReadinessProbe();
+    return this;
+  }
+
+  async stop() {
+    if (!this.running && this.connections.size === 0) return;
+    this.running = false;
+    this.upstreamReady = this.upstreamUrl === null;
+    if (this.readinessProbe) {
+      this.readinessProbe.intentionalClose = true;
+      this.readinessProbe.socket?.terminate();
+      clearTimeout(this.readinessProbe.timer);
+      this.readinessProbe = null;
+    }
+    for (const connection of this.connections.values()) closeSocket(connection.socket, 1001, 'service stopping');
+    for (const connection of this.forwardConnections) connection.upstream?.terminate();
+    this.connections.clear();
+    this.forwardConnections.clear();
+    if (this.bridge?.started) await this.bridge.stop();
+    if (this.transport.running) await this.transport.stop();
+  }
+
+  status() {
+    const bridgeReady = this.upstreamUrl !== null || (this.bridge?.started === true && this.transport.running === true);
+    const upstreamReady = this.upstreamUrl === null || this.upstreamReady;
+    const checks = [
+      { check: 'federation_runtime_started', ok: this.running === true },
+      { check: 'federation_storage', ok: this.storageReady === true },
+      { check: 'federation_authenticator', ok: this.authReady === true },
+      { check: 'federation_bridge', ok: bridgeReady },
+      { check: 'federation_upstream', ok: upstreamReady },
+    ];
+    return {
+      enabled: true,
+      ready: checks.every((check) => check.ok),
+      path: this.path,
+      transport: 'authenticated_websocket_binary',
+      storage_ref: this.storage.ref,
+      durable_storage: this.storage.durable === true,
+      auth_ref: this.authRef,
+      kms_ref_configured: nonEmptyString(this.kmsRef),
+      active_connections: this.connections.size,
+      checks,
+      last_dependency_error: this.lastDependencyError,
+    };
+  }
+
+  async persist(direction, peerId, decoded, frame) {
+    try {
+      await this.storage.appendOpaqueFrame(storageRecord(direction, peerId, decoded, frame));
+    } catch (error) {
+      this.storageReady = false;
+      this.lastDependencyError = error instanceof Error ? error.message : 'federation storage write failed';
+      throw error;
+    }
+  }
+
+  async handleUpgrade(request, socket, head) {
+    let pathname;
+    try {
+      pathname = new URL(request.url ?? '/', 'http://enigma.local').pathname;
+    } catch {
+      rejectUpgrade(socket, 400, 'invalid request path');
+      return;
+    }
+    if (pathname !== this.path) {
+      rejectUpgrade(socket, 404, 'WebSocket path is not available');
+      return;
+    }
+    if (!this.running || !this.storageReady || !this.authReady) {
+      rejectUpgrade(socket, 503, 'federation service is not ready');
+      return;
+    }
+    let principal;
+    try {
+      principal = await authenticateRequest(this.authenticator, request);
+    } catch {
+      rejectUpgrade(socket, 401, 'federation authentication failed');
+      return;
+    }
+    this.webSocketServer.handleUpgrade(request, socket, head, (webSocket) => {
+      this.acceptConnection(webSocket, principal, request);
+    });
+  }
+
+  acceptConnection(socket, principal, request) {
+    const existing = this.connections.get(principal.peerId);
+    if (existing) closeSocket(existing.socket, 4009, 'authenticated peer reconnected');
+    const connection = { socket, principal, request, upstream: null, queue: [], queuedBytes: 0, reconnects: 0, closed: false };
+    this.connections.set(principal.peerId, connection);
+    socket.on('message', (data, isBinary) => void this.handleClientFrame(connection, data, isBinary));
+    socket.once('close', () => {
+      connection.closed = true;
+      connection.upstream?.terminate();
+      this.forwardConnections.delete(connection);
+      if (this.connections.get(principal.peerId) === connection) this.connections.delete(principal.peerId);
+    });
+    socket.on('error', () => undefined);
+    if (this.upstreamUrl) this.connectUpstream(connection);
+  }
+
+  async handleClientFrame(connection, data, isBinary) {
+    try {
+      if (!receivedBinaryFrame(data, isBinary)) throw new Error('federation WebSocket accepts binary frames only');
+      const { frame, decoded } = validateFederationFrame(data, this.maxFrameBytes, connection.principal.publicKey);
+      await this.persist('ingress', connection.principal.peerId, decoded, frame);
+      if (this.upstreamUrl) {
+        if (connection.upstream?.readyState === WebSocket.OPEN) {
+          const sent = await sendBinary(connection.upstream, frame, this.maxBufferedBytes);
+          if (!sent) throw new Error('federation upstream backpressure limit exceeded');
+        } else {
+          this.enqueueFrame(connection, frame);
+        }
+      } else {
+        this.transport.emit('packet', {
+          packetBuffer: frame,
+          fromPeerId: connection.principal.peerId,
+          transportId: this.transport.id,
+        });
+      }
+    } catch (error) {
+      this.lastDependencyError = error instanceof Error ? error.message : 'invalid federation client frame';
+      this.emit('frameRejected', { direction: 'ingress', peerId: connection.principal.peerId, error });
+      closeSocket(connection.socket, 1008, 'invalid federation frame');
+    }
+  }
+
+  enqueueFrame(connection, frame) {
+    if (connection.queue.length >= this.maxQueuedFrames || connection.queuedBytes + frame.length > this.maxQueuedBytes) {
+      throw new Error('federation reconnect queue limit exceeded');
+    }
+    connection.queue.push(Buffer.from(frame));
+    connection.queuedBytes += frame.length;
+  }
+
+  connectUpstream(connection) {
+    if (connection.closed) return;
+    const headers = {
+      authorization: headerValue(connection.request.headers, 'authorization'),
+      'x-enigma-peer-id': connection.principal.peerId,
+      'x-enigma-public-key': connection.principal.publicKey,
+    };
+    const upstream = new WebSocket(this.upstreamUrl, {
+      headers,
+      maxPayload: this.maxFrameBytes,
+      perMessageDeflate: false,
+      handshakeTimeout: this.connectTimeoutMs,
+    });
+    connection.upstream = upstream;
+    this.forwardConnections.add(connection);
+    upstream.once('open', async () => {
+      connection.reconnects = 0;
+      this.upstreamReady = true;
+      try {
+        while (connection.queue.length > 0) {
+          const frame = connection.queue.shift();
+          connection.queuedBytes -= frame.length;
+          if (!await sendBinary(upstream, frame, this.maxBufferedBytes)) {
+            throw new Error('federation upstream backpressure limit exceeded');
+          }
+        }
+      } catch {
+        closeSocket(connection.socket, 1013, 'federation upstream backpressure');
+      }
+    });
+    upstream.on('message', (data, isBinary) => void this.handleUpstreamFrame(connection, data, isBinary));
+    upstream.on('error', () => undefined);
+    upstream.once('close', () => {
+      if (connection.upstream === upstream) connection.upstream = null;
+      this.upstreamReady = [...this.forwardConnections].some((item) => item.upstream?.readyState === WebSocket.OPEN) || this.readinessProbe?.socket?.readyState === WebSocket.OPEN;
+      if (connection.closed) return;
+      connection.reconnects += 1;
+      if (connection.reconnects > this.reconnectAttempts) {
+        closeSocket(connection.socket, 1013, 'federation upstream unavailable');
+        return;
+      }
+      setTimeout(() => this.connectUpstream(connection), this.reconnectDelayMs * connection.reconnects).unref?.();
+    });
+  }
+
+  async handleUpstreamFrame(connection, data, isBinary) {
+    try {
+      if (!receivedBinaryFrame(data, isBinary)) throw new Error('federation upstream sent a non-binary frame');
+      const { frame, decoded } = validateFederationFrame(data, this.maxFrameBytes);
+      if (this.expectedUpstreamPublicKeys.size > 0 && !this.expectedUpstreamPublicKeys.has(decoded.senderPubkey.toString('hex'))) {
+        throw new Error('federation upstream sender is not trusted');
+      }
+      await this.persist('egress', connection.principal.peerId, decoded, frame);
+      if (!await sendBinary(connection.socket, frame, this.maxBufferedBytes)) {
+        throw new Error('federation client backpressure limit exceeded');
+      }
+    } catch (error) {
+      this.lastDependencyError = error instanceof Error ? error.message : 'invalid federation upstream frame';
+      this.emit('frameRejected', { direction: 'upstream', peerId: connection.principal.peerId, error });
+      closeSocket(connection.socket, 1008, 'invalid federation upstream frame');
+    }
+  }
+
+  async sendToPeer(packetBuffer, peerId) {
+    const connection = this.connections.get(peerId);
+    if (!connection) return false;
+    const { frame, decoded } = validateFederationFrame(packetBuffer, this.maxFrameBytes);
+    await this.persist('egress', peerId, decoded, frame);
+    return await sendBinary(connection.socket, frame, this.maxBufferedBytes);
+  }
+
+  startReadinessProbe() {
+    if (!this.upstreamUrl || !this.upstreamHeaders || this.readinessProbe) return;
+    const probe = { socket: null, timer: null, intentionalClose: false };
+    this.readinessProbe = probe;
+    const connect = () => {
+      if (!this.running || probe.intentionalClose) return;
+      const socket = new WebSocket(this.upstreamUrl, {
+        headers: this.upstreamHeaders,
+        maxPayload: this.maxFrameBytes,
+        perMessageDeflate: false,
+        handshakeTimeout: this.connectTimeoutMs,
+      });
+      probe.socket = socket;
+      socket.once('open', () => { this.upstreamReady = true; });
+      socket.on('message', () => closeSocket(socket, 1008, 'readiness connection cannot carry frames'));
+      socket.on('error', () => undefined);
+      socket.once('close', () => {
+        if (probe.intentionalClose) return;
+        this.upstreamReady = [...this.forwardConnections].some((item) => item.upstream?.readyState === WebSocket.OPEN);
+        probe.timer = setTimeout(connect, this.reconnectDelayMs);
+        probe.timer.unref?.();
+      });
+    };
+    connect();
+  }
+}
+
+export function attachFederationServeMode(server, options) {
+  return new FederationServeRuntime(server, options);
+}