@dxos/edge-client 0.10.0 → 0.11.1

package/src/edge-ws-connection.test.ts

@@ -0,0 +1,219 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { describe, onTestFinished, test, vi } from 'vitest';
+
+import { Trigger } from '@dxos/async';
+import { invariant } from '@dxos/invariant';
+import { bufWkt } from '@dxos/protocols/buf';
+import { type Message, TextMessageSchema } from '@dxos/protocols/buf/dxos/edge/messenger_pb';
+
+import { protocol } from './defs';
+import { type EdgeIdentity } from './edge-identity';
+import { WebSocketMuxer } from './edge-ws-muxer';
+
+// Segmented-message chunk count depends on the protobuf envelope overhead, which is
+// determined empirically (see chunk-count assertions below) rather than assumed.
+const MAX_CHUNK_LENGTH = 64;
+const MESSAGE_A_CONTENT = 'a'.repeat(20);
+const MESSAGE_B_CONTENT = 'b'.repeat(120);
+
+// Replace isomorphic-ws's default export with a controllable fake so tests can drive
+// the connection's onmessage handler directly and inspect what it sends.
+const { FakeWebSocket } = vi.hoisted(() => {
+  class FakeWebSocket {
+    static instances: FakeWebSocket[] = [];
+
+    readyState = 1;
+    protocol = '';
+    binaryType = 'nodebuffer';
+    onopen: (() => void) | null = null;
+    onclose: ((event: { code?: number; reason?: string }) => void) | null = null;
+    onerror: ((event: { error?: unknown; message?: string }) => void) | null = null;
+    onmessage: ((event: { data: unknown; type: string }) => void) | null = null;
+    readonly sent: unknown[] = [];
+
+    constructor(
+      public readonly url: string,
+      public readonly protocols?: string[],
+      public readonly options?: unknown,
+    ) {
+      FakeWebSocket.instances.push(this);
+    }
+
+    send(data: unknown): void {
+      this.sent.push(data);
+    }
+
+    close(): void {}
+  }
+
+  return { FakeWebSocket };
+});
+
+vi.mock('isomorphic-ws', () => ({ default: FakeWebSocket }));
+
+const { EdgeWsConnection } = await import('./edge-ws-connection');
+
+const testIdentity: EdgeIdentity = {
+  peerKey: 'test-peer-key',
+  identityDid: 'did:halo:test',
+  presentCredentials: async () => {
+    throw new Error('not implemented');
+  },
+};
+
+describe('EdgeWsConnection', () => {
+  test('reassembles segmented messages delivered as Blobs out of order', async ({ expect }) => {
+    const [chunksA, chunksB] = await buildSegmentedChunks([MESSAGE_A_CONTENT, MESSAGE_B_CONTENT]);
+    expect(chunksA).toHaveLength(2);
+    expect(chunksB).toHaveLength(4);
+
+    const { ws, received, allReceived } = await openTestConnection(2);
+
+    const deferredA = chunksA.map(deferredChunk);
+    const deferredB = chunksB.map(deferredChunk);
+
+    // Dispatch every chunk in correct wire order, as a real WebSocket would.
+    for (const { blob } of [...deferredA, ...deferredB]) {
+      ws.onmessage?.({ data: blob, type: 'message' });
+    }
+
+    // Resolve the underlying `arrayBuffer()` reads out of order, simulating the browser
+    // race where concurrent Blob reads do not complete in arrival order.
+    deferredA[0].resolve();
+    deferredB[0].resolve();
+    deferredB[1].resolve();
+    deferredA[1].resolve();
+    deferredB[2].resolve();
+    deferredB[3].resolve();
+
+    await allReceived.wait();
+    expect(received).toHaveLength(2);
+
+    const [messageA, messageB] = received;
+    invariant(messageA.payload);
+    invariant(messageB.payload);
+    expect(bufWkt.anyUnpack(messageA.payload, TextMessageSchema)?.message).toStrictEqual(MESSAGE_A_CONTENT);
+    expect(bufWkt.anyUnpack(messageB.payload, TextMessageSchema)?.message).toStrictEqual(MESSAGE_B_CONTENT);
+  });
+
+  test('reassembles segmented messages delivered as ArrayBuffers', async ({ expect }) => {
+    const [chunksA, chunksB] = await buildSegmentedChunks([MESSAGE_A_CONTENT, MESSAGE_B_CONTENT]);
+
+    const { ws, received, allReceived } = await openTestConnection(2);
+
+    for (const chunk of [...chunksA, ...chunksB]) {
+      ws.onmessage?.({ data: toArrayBuffer(chunk), type: 'message' });
+    }
+
+    await allReceived.wait();
+    expect(received).toHaveLength(2);
+
+    const [messageA, messageB] = received;
+    invariant(messageA.payload);
+    invariant(messageB.payload);
+    expect(bufWkt.anyUnpack(messageA.payload, TextMessageSchema)?.message).toStrictEqual(MESSAGE_A_CONTENT);
+    expect(bufWkt.anyUnpack(messageB.payload, TextMessageSchema)?.message).toStrictEqual(MESSAGE_B_CONTENT);
+  });
+});
+
+/**
+ * A Blob whose `arrayBuffer()` resolves only when `resolve()` is called, so tests can
+ * control the order in which concurrent `blob.arrayBuffer()` reads complete.
+ */
+class DeferredBlob extends Blob {
+  readonly #result: Promise<ArrayBuffer>;
+
+  constructor(result: Promise<ArrayBuffer>) {
+    super();
+    this.#result = result;
+  }
+
+  override async arrayBuffer(): Promise<ArrayBuffer> {
+    // `await` (not a bare `return`) so the listener attaches to `#result` synchronously,
+    // during dispatch, rather than one microtask later — otherwise resolving out of
+    // dispatch order has no effect once every deferred promise is already settled.
+    return await this.#result;
+  }
+}
+
+type DeferredChunk = {
+  blob: DeferredBlob;
+  resolve: () => void;
+};
+
+const deferredChunk = (bytes: Uint8Array): DeferredChunk => {
+  let resolveResult: (buffer: ArrayBuffer) => void = () => {};
+  const result = new Promise<ArrayBuffer>((res) => {
+    resolveResult = res;
+  });
+  // Copy so the returned ArrayBuffer starts at offset 0 and is independent of the source view.
+  const copy = new Uint8Array(bytes);
+  return { blob: new DeferredBlob(result), resolve: () => resolveResult(copy.buffer) };
+};
+
+const toArrayBuffer = (bytes: Uint8Array): ArrayBuffer => new Uint8Array(bytes).buffer;
+
+const textMessage = (message: string) =>
+  protocol.createMessage(TextMessageSchema, {
+    serviceId: 'test-service',
+    payload: { message },
+  });
+
+/**
+ * Sends each content string as a message on the same muxer/channel and returns the wire
+ * chunks grouped by message, mirroring how `WebSocketMuxer` splits segmented messages.
+ */
+const buildSegmentedChunks = async (contents: string[]): Promise<Uint8Array[][]> => {
+  const sentMessages: Uint8Array[] = [];
+  const muxer = new WebSocketMuxer(
+    { readyState: 1, send: (message: string) => sentMessages.push(Buffer.from(message)) },
+    { maxChunkLength: MAX_CHUNK_LENGTH },
+  );
+
+  const chunkCounts: number[] = [];
+  for (const content of contents) {
+    const before = sentMessages.length;
+    await muxer.send(textMessage(content));
+    chunkCounts.push(sentMessages.length - before);
+  }
+
+  const chunksByMessage: Uint8Array[][] = [];
+  let offset = 0;
+  for (const count of chunkCounts) {
+    chunksByMessage.push(sentMessages.slice(offset, offset + count));
+    offset += count;
+  }
+  return chunksByMessage;
+};
+
+const openTestConnection = async (expectedMessages: number) => {
+  const received: Message[] = [];
+  const allReceived = new Trigger();
+  const connection = new EdgeWsConnection(
+    testIdentity,
+    { url: new URL('ws://localhost:1234') },
+    {
+      onConnected: () => {},
+      onMessage: (message) => {
+        received.push(message);
+        if (received.length === expectedMessages) {
+          allReceived.wake();
+        }
+      },
+      onRestartRequired: () => {},
+    },
+  );
+  await connection.open();
+  onTestFinished(async () => {
+    await connection.close();
+  });
+
+  const ws = FakeWebSocket.instances.at(-1);
+  invariant(ws, 'FakeWebSocket instance not created');
+  ws.onopen?.();
+
+  return { connection, ws, received, allReceived };
+};

package/src/edge-ws-connection.ts

@@ -4,7 +4,7 @@
 
 import WebSocket from 'isomorphic-ws';
 
-import { scheduleTask, scheduleTaskInterval } from '@dxos/async';
+import { Mutex, scheduleTask, scheduleTaskInterval } from '@dxos/async';
 import { Context, Resource } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { log, logInfo } from '@dxos/log';
@@ -19,6 +19,13 @@ import { toUint8Array } from './protocol';
 
 const SIGNAL_KEEPALIVE_INTERVAL = 4_000;
 const SIGNAL_KEEPALIVE_TIMEOUT = 12_000;
+/**
+ * Watchdog self-check: if the inactivity timer fires later than its schedule by more than this,
+ * the local event loop was starved (heavy WASM sync compute pins it for seconds at a time) —
+ * our pings were not being sent and inbound pongs were not being processed, so the silence says
+ * nothing about the connection. Probe and re-arm instead of restarting.
+ */
+const KEEPALIVE_WATCHDOG_LATE_TOLERANCE = 3_000;
 
 export type EdgeWsConnectionCallbacks = {
   onConnected: () => void;
@@ -36,6 +43,7 @@ export class EdgeWsConnection extends Resource {
 
   // Latency tracking.
   private _pingTimestamp: number | undefined;
+  private _lastPingSentTimestamp = 0;
   private _rtt = 0;
 
   // Rate tracking with sliding window.
@@ -48,6 +56,15 @@ export class EdgeWsConnection extends Resource {
   private _messagesSent = 0;
   private _messagesReceived = 0;
 
+  /**
+   * WebSocket frames arrive in order, but converting frame data to bytes is async
+   * (the `Blob` fallback path awaits `blob.arrayBuffer()`), and concurrent conversions
+   * are not guaranteed to complete in arrival order. Segmented-message reassembly in
+   * `WebSocketMuxer` requires chunks to reach `receiveData` in arrival order, so message
+   * processing is serialized through this lock.
+   */
+  private readonly _receiveMutex = new Mutex();
+
   constructor(
     private readonly _identity: EdgeIdentity,
     private readonly _connectionInfo: { url: URL; protocolHeader?: string; headers?: Record<string, string> },
@@ -123,6 +140,10 @@ export class EdgeWsConnection extends Resource {
         : [...baseProtocols],
       this._connectionInfo.headers ? { headers: this._connectionInfo.headers } : undefined,
     );
+    // Deliver frame data as `ArrayBuffer` rather than `Blob` so bytes are available
+    // synchronously; avoids the async `blob.arrayBuffer()` reads that can otherwise
+    // complete out of arrival order (see `_receiveChain`).
+    this._ws.binaryType = 'arraybuffer';
     const muxer = new WebSocketMuxer(this._ws);
     this._wsMuxer = muxer;
 
@@ -155,7 +176,7 @@ export class EdgeWsConnection extends Resource {
     /**
      * https://developer.mozilla.org/en-US/docs/Web/API/MessageEvent/data
      */
-    this._ws.onmessage = async (event: WebSocket.MessageEvent) => {
+    this._ws.onmessage = (event: WebSocket.MessageEvent) => {
       if (!this.isOpen) {
         log.verbose('message ignored on closed connection', { event: event.type });
         return;
@@ -170,23 +191,34 @@ export class EdgeWsConnection extends Resource {
         this._rescheduleHeartbeatTimeout();
         return;
       }
-      const bytes = await toUint8Array(event.data);
-      this._recordBytes(0, bytes.byteLength);
-      if (!this.isOpen) {
-        return;
-      }
 
-      this._messagesReceived++;
+      // `_receiveMessage` serializes on `_receiveMutex`; `acquire` enqueues synchronously,
+      // so locks are taken in arrival order regardless of async conversion timing.
+      void this._receiveMessage(event.data, muxer).catch((err) => log.catch(err));
+    };
+  }
+
+  private async _receiveMessage(data: WebSocket.Data, muxer: WebSocketMuxer): Promise<void> {
+    // Serialize processing so bytes reach `muxer.receiveData` in arrival order. The guard
+    // releases on scope exit even if processing throws, so a single bad message is logged
+    // and dropped instead of stalling every message queued after it.
+    using _guard = await this._receiveMutex.acquire();
+    const bytes = await toUint8Array(data);
+    this._recordBytes(0, bytes.byteLength);
+    if (!this.isOpen) {
+      return;
+    }
 
-      const message = this._ws?.protocol?.includes(EdgeWebsocketProtocol.V0)
-        ? buf.fromBinary(MessageSchema, bytes)
-        : muxer.receiveData(bytes);
+    this._messagesReceived++;
 
-      if (message) {
-        log('received', { from: message.source, payload: protocol.getPayloadType(message) });
-        this._callbacks.onMessage(message);
-      }
-    };
+    const message = this._ws?.protocol?.includes(EdgeWebsocketProtocol.V0)
+      ? buf.fromBinary(MessageSchema, bytes)
+      : muxer.receiveData(bytes);
+
+    if (message) {
+      log('received', { from: message.source, payload: protocol.getPayloadType(message) });
+      this._callbacks.onMessage(message);
+    }
   }
 
   protected override async _close(): Promise<void> {
@@ -212,35 +244,72 @@ export class EdgeWsConnection extends Resource {
       async () => {
         // TODO(mykola): use RFC6455 ping/pong once implemented in the browser?
         // Cloudflare's worker responds to this `without interrupting hibernation`. https://developers.cloudflare.com/durable-objects/api/websockets/#setwebsocketautoresponse
-        this._pingTimestamp = Date.now();
-        this._ws?.send('__ping__');
+        this._sendPing();
       },
       SIGNAL_KEEPALIVE_INTERVAL,
     );
+    this._sendPing();
+    this._rescheduleHeartbeatTimeout();
+  }
+
+  private _sendPing(): void {
+    if (!this._ws) {
+      return;
+    }
     this._pingTimestamp = Date.now();
+    this._lastPingSentTimestamp = Date.now();
     this._ws.send('__ping__');
-    this._rescheduleHeartbeatTimeout();
   }
 
+  /**
+   * Inactivity watchdog. Restarts the connection only after a fair trial: pings were actually
+   * flowing (a recent send), the timer fired on schedule (the local event loop was alive to
+   * process an answer), and still nothing was received for the full window. Wall-clock silence
+   * alone is not evidence — sync compute can pin the event loop for seconds, during which the
+   * ping sender does not run and arrived pongs are not processed; restarting a healthy
+   * connection on that basis costs a re-handshake and fails in-flight sync rounds.
+   */
   private _rescheduleHeartbeatTimeout(): void {
     if (!this.isOpen) {
       return;
     }
     void this._inactivityTimeoutCtx?.dispose();
     this._inactivityTimeoutCtx = new Context();
+    const armedAt = Date.now();
     scheduleTask(
       this._inactivityTimeoutCtx,
       () => {
-        if (this.isOpen) {
-          if (Date.now() - this._lastReceivedMessageTimestamp > SIGNAL_KEEPALIVE_TIMEOUT) {
-            log.warn('restart due to inactivity timeout', {
-              lastReceivedMessageTimestamp: this._lastReceivedMessageTimestamp,
-            });
-            this._callbacks.onRestartRequired();
-          } else {
-            this._rescheduleHeartbeatTimeout();
-          }
+        if (!this.isOpen) {
+          return;
+        }
+        const now = Date.now();
+        const silenceMs = now - this._lastReceivedMessageTimestamp;
+        if (silenceMs <= SIGNAL_KEEPALIVE_TIMEOUT) {
+          this._rescheduleHeartbeatTimeout();
+          return;
         }
+        const pingAgeMs = this._lastPingSentTimestamp ? now - this._lastPingSentTimestamp : Number.POSITIVE_INFINITY;
+        const firedLateByMs = now - armedAt - SIGNAL_KEEPALIVE_TIMEOUT;
+        const pingsWereFlowing = pingAgeMs <= SIGNAL_KEEPALIVE_INTERVAL * 2;
+        const loopWasLive = firedLateByMs < KEEPALIVE_WATCHDOG_LATE_TOLERANCE;
+        if (pingsWereFlowing && loopWasLive) {
+          log.warn('restart due to inactivity timeout', {
+            silenceMs,
+            pingAgeMs,
+            lastReceivedMessageTimestamp: this._lastReceivedMessageTimestamp,
+          });
+          this._callbacks.onRestartRequired();
+          return;
+        }
+        // The silence is self-inflicted (starved event loop stopped our pings and delayed this
+        // timer). Probe immediately and give the connection a fresh full window to answer.
+        log.verbose('keepalive starved by event loop; probing instead of restarting', {
+          silenceMs,
+          pingAgeMs,
+          firedLateByMs,
+        });
+        this._sendPing();
+        this._rescheduleHeartbeatTimeout();
       },
       SIGNAL_KEEPALIVE_TIMEOUT,
     );

package/src/index.ts

@@ -17,3 +17,4 @@ export * from './hub-http-client';
 export * from './edge-identity';
 export * from './edge-ws-muxer';
 export * from './http-client';
+export * from './browser-rendering';

package/src/protocol.ts

@@ -69,11 +69,14 @@ export class Protocol {
     {
       source,
       target,
+      tags,
       payload,
       serviceId,
     }: {
       source?: PeerData;
       target?: PeerData[];
+      // Broadcast tags (DX-1125). Set with no resolvable `target` to publish to the swarm.
+      tags?: string[];
       payload?: buf.MessageInitShape<Desc>;
       serviceId?: string;
     },
@@ -82,6 +85,7 @@ export class Protocol {
       timestamp: new Date().toISOString(),
       source,
       target,
+      tags,
       serviceId,
       payload: payload ? bufWkt.anyPack(type, buf.create(type, payload)) : undefined,
     });
@@ -97,9 +101,14 @@ export const toUint8Array = async (data: any): Promise<Uint8Array> => {
     return bufferToArray(data);
   }
 
-  // Browser.
+  // Browser with `binaryType = 'arraybuffer'`.
+  if (data instanceof ArrayBuffer) {
+    return new Uint8Array(data);
+  }
+
+  // Browser fallback (`binaryType = 'blob'`, the WebSocket default).
   if (data instanceof Blob) {
-    return new Uint8Array(await (data as Blob).arrayBuffer());
+    return new Uint8Array(await data.arrayBuffer());
   }
 
   throw new Error(`Unexpected datatype: ${data}`);

package/src/testing/test-utils.ts

@@ -62,7 +62,11 @@ export const createTestEdgeWsServer = async (port = DEFAULT_PORT, params?: TestE
     });
 
     ws.on('close', () => {
-      connection = undefined;
+      // During a reconnect the new connection may be admitted before the old
+      // socket's close event fires; only clear if this socket is still current.
+      if (connection?.ws === ws) {
+        connection = undefined;
+      }
       closeTrigger.wake();
     });
   });

package/dist/lib/neutral/chunk-J5LGTIGS.mjs

@@ -1,10 +0,0 @@
-var __defProp = Object.defineProperty;
-var __export = (target, all) => {
-  for (var name in all)
-    __defProp(target, name, { get: all[name], enumerable: true });
-};
-
-export {
-  __export
-};
-//# sourceMappingURL=chunk-J5LGTIGS.mjs.map

package/dist/lib/neutral/chunk-J5LGTIGS.mjs.map

@@ -1,7 +0,0 @@
-{
-  "version": 3,
-  "sources": [],
-  "sourcesContent": [],
-  "mappings": "",
-  "names": []
-}