@dxos/network-manager 0.1.55 → 0.1.56-main.09ca29d

package/src/swarm/peer.ts

@@ -2,10 +2,10 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { scheduleTask, synchronized } from '@dxos/async';
 import { Context } from '@dxos/context';
+import { CancelledError } from '@dxos/errors';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { Answer } from '@dxos/protocols/proto/dxos/mesh/swarm';
@@ -14,6 +14,7 @@ import { OfferMessage, SignalMessage, SignalMessenger } from '../signal';
 import { TransportFactory } from '../transport';
 import { WireProtocolProvider } from '../wire-protocol';
 import { Connection, ConnectionState } from './connection';
+import { ConnectionLimiter } from './connection-limiter';
 
 interface PeerCallbacks {
   /**
@@ -80,7 +81,6 @@ export class Peer {
   public advertizing = false;
 
   public initiating = false;
-
   constructor(
     public readonly id: PublicKey,
     public readonly topic: PublicKey,
@@ -88,6 +88,7 @@ export class Peer {
     private readonly _signalMessaging: SignalMessenger,
     private readonly _protocolProvider: WireProtocolProvider,
     private readonly _transportFactory: TransportFactory,
+    private readonly _connectionLimiter: ConnectionLimiter,
     private readonly _callbacks: PeerCallbacks,
   ) {}
 
@@ -102,10 +103,11 @@ export class Peer {
       // Peer with the highest Id closes its connection, and accepts remote peer's offer.
       if (remoteId.toHex() < this.localPeerId.toHex()) {
         // TODO(burdon): Too verbose.
-        log("closing local connection and accepting remote peer's offer", {
-          id: this.id,
+        log('close local connection', {
+          localPeerId: this.id,
           topic: this.topic,
-          peerId: this.localPeerId,
+          remotePeerId: this.localPeerId,
+          sessionId: this.connection?.sessionId,
         });
 
         if (this.connection) {
@@ -123,10 +125,15 @@ export class Peer {
         // Connection might have been already established.
         invariant(message.sessionId);
         const connection = this._createConnection(false, message.sessionId);
+
         try {
-          connection.openConnection();
+          await this._connectionLimiter.connecting(message.sessionId);
+          await connection.openConnection();
         } catch (err: any) {
-          log.warn('connection error', { topic: this.topic, peerId: this.localPeerId, remoteId: this.id, err });
+          if (!(err instanceof CancelledError)) {
+            log.warn('connection error', { topic: this.topic, peerId: this.localPeerId, remoteId: this.id, err });
+          }
+
           // Calls `onStateChange` with CLOSED state.
           await this.closeConnection();
         }
@@ -145,10 +152,13 @@ export class Peer {
     invariant(!this.connection, 'Already connected.');
     const sessionId = PublicKey.random();
     log('initiating...', { id: this.id, topic: this.topic, peerId: this.id, sessionId });
+
     const connection = this._createConnection(true, sessionId);
     this.initiating = true;
 
     try {
+      await this._connectionLimiter.connecting(sessionId);
+
       const answer = await this._signalMessaging.offer({
         author: this.localPeerId,
         recipient: this.id,
@@ -166,7 +176,7 @@ export class Peer {
         this._callbacks.onRejected();
         return;
       }
-      connection.openConnection();
+      await connection.openConnection();
       this._callbacks.onAccepted();
     } catch (err: any) {
       log('initiation error', { err, topic: this.topic, peerId: this.localPeerId, remoteId: this.id });
@@ -214,6 +224,15 @@ export class Peer {
           this.availableToConnect = true;
           this._lastConnectionTime = Date.now();
           this._callbacks.onConnected();
+
+          this._connectionLimiter.doneConnecting(sessionId);
+          log.trace('dxos.mesh.connection.connected', {
+            topic: this.topic,
+            localPeerId: this.localPeerId,
+            remotePeerId: this.id,
+            sessionId,
+            initiator,
+          });
           break;
         }
 
@@ -221,6 +240,14 @@ export class Peer {
           log('connection closed', { topic: this.topic, peerId: this.localPeerId, remoteId: this.id, initiator });
           invariant(this.connection === connection, 'Connection mismatch (race condition).');
 
+          log.trace('dxos.mesh.connection.closed', {
+            topic: this.topic,
+            localPeerId: this.localPeerId,
+            remotePeerId: this.id,
+            sessionId,
+            initiator,
+          });
+
           if (this._lastConnectionTime && this._lastConnectionTime + CONNECTION_COUNTS_STABLE_AFTER < Date.now()) {
             // If we're closing the connection, and it has been connected for a while, reset the backoff.
             this._availableAfter = 0;
@@ -231,6 +258,7 @@ export class Peer {
 
           this.connection = undefined;
           this._callbacks.onDisconnected();
+          this._connectionLimiter.doneConnecting(sessionId);
 
           scheduleTask(
             this._connectionCtx!,
@@ -247,6 +275,14 @@ export class Peer {
     });
     connection.errors.handle((err) => {
       log.warn('connection error', { topic: this.topic, peerId: this.localPeerId, remoteId: this.id, initiator, err });
+      log.trace('dxos.mesh.connection.error', {
+        topic: this.topic,
+        localPeerId: this.localPeerId,
+        remotePeerId: this.id,
+        sessionId,
+        initiator,
+        err,
+      });
 
       // Calls `onStateChange` with CLOSED state.
       void this.closeConnection();
@@ -276,6 +312,7 @@ export class Peer {
       log('dropping signal message for non-existent connection', { message });
       return;
     }
+
     await this.connection.signal(message);
   }
 

package/src/swarm/swarm.test.ts

@@ -2,31 +2,43 @@
 // Copyright 2020 DXOS.org
 //
 
-// @dxos/test platform=nodejs
-
-import { expect } from 'earljs';
+import { expect } from 'chai';
 
 import { asyncTimeout, sleep } from '@dxos/async';
 import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { MemorySignalManager, MemorySignalManagerContext, Messenger } from '@dxos/messaging';
-import { afterTest, beforeEach, describe, test } from '@dxos/test';
+import { MemorySignalManager, MemorySignalManagerContext, Messenger, SignalManager } from '@dxos/messaging';
+import { afterTest, describe, test } from '@dxos/test';
+import { ComplexSet } from '@dxos/util';
 
 import { TestWireProtocol } from '../testing/test-wire-protocol';
 import { FullyConnectedTopology } from '../topology';
 import { createWebRTCTransportFactory } from '../transport';
+import { ConnectionState } from './connection';
+import { ConnectionLimiter } from './connection-limiter';
 import { Swarm } from './swarm';
 
+type TestPeer = {
+  swarm: Swarm;
+  peerId: PublicKey;
+  protocol: TestWireProtocol;
+  topic: PublicKey;
+  signalManager: SignalManager;
+};
+
 describe('Swarm', () => {
   const context = new MemorySignalManagerContext();
-  let signalManager: MemorySignalManager;
-
-  beforeEach(() => {
-    signalManager = new MemorySignalManager(context);
-    afterTest(() => signalManager.close());
-  });
 
-  const setupSwarm = async ({ topic, peerId }: { topic: PublicKey; peerId: PublicKey }) => {
+  const setupSwarm = async ({
+    topic = PublicKey.random(),
+    peerId = PublicKey.random(),
+    connectionLimiter = new ConnectionLimiter(),
+    signalManager = new MemorySignalManager(context),
+  }: {
+    topic?: PublicKey;
+    peerId?: PublicKey;
+    connectionLimiter?: ConnectionLimiter;
+    signalManager?: SignalManager;
+  }): Promise<TestPeer> => {
     const protocol = new TestWireProtocol(peerId);
     const swarm = new Swarm(
       topic,
@@ -36,116 +48,154 @@ describe('Swarm', () => {
       new Messenger({ signalManager }),
       createWebRTCTransportFactory(),
       undefined,
+      connectionLimiter,
     );
 
     afterTest(async () => {
       await swarm.destroy();
+      await signalManager.close();
     });
 
     await swarm.open();
 
-    return { swarm, protocol };
+    return { swarm, protocol, topic, peerId, signalManager };
   };
 
   test('connects two peers in a swarm', async () => {
     const topic = PublicKey.random();
-    const peerId1 = PublicKey.random();
-    const peerId2 = PublicKey.random();
 
-    const { swarm: swarm1, protocol: protocol1 } = await setupSwarm({ topic, peerId: peerId1 });
+    const peer1 = await setupSwarm({ topic });
+    const peer2 = await setupSwarm({ topic });
 
-    const { swarm: swarm2 } = await setupSwarm({ topic, peerId: peerId2 });
+    expect(peer1.swarm.connections.length).to.equal(0);
+    expect(peer2.swarm.connections.length).to.equal(0);
 
-    expect(swarm1.connections.length).toEqual(0);
-    expect(swarm2.connections.length).toEqual(0);
-
-    const promise = Promise.all([
-      asyncTimeout(swarm1.connected.waitForCount(1), 3000, new Error('Swarm1 connect timeout.')),
-      asyncTimeout(swarm2.connected.waitForCount(1), 3000, new Error('Swarm2 connect timeout.')),
-    ]);
+    await connectSwarms(peer1, peer2);
+  }).timeout(5_000);
 
-    // Behavior of the Signal Server.
-    swarm1.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId1.asUint8Array(),
-        since: new Date(),
-      },
-    });
+  test('two peers try to originate connections to each other simultaneously', async () => {
+    const topic = PublicKey.random();
 
-    swarm1.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId2.asUint8Array(),
-        since: new Date(),
-      },
-    });
+    const peer1 = await setupSwarm({ topic });
+    const peer2 = await setupSwarm({ topic });
 
-    log('Candidates changed');
-    await promise;
-    log('Swarms connected');
+    expect(peer1.swarm.connections.length).to.equal(0);
+    expect(peer2.swarm.connections.length).to.equal(0);
 
-    await protocol1.testConnection(peerId2);
+    await connectSwarms(peer1, peer2);
   }).timeout(5_000);
 
-  test('two peers try to originate connections to each other simultaneously', async () => {
+  test('second peer discovered after delay', async () => {
     const topic = PublicKey.random();
-    const peerId1 = PublicKey.random();
-    const peerId2 = PublicKey.random();
 
-    const { swarm: swarm1 } = await setupSwarm({ topic, peerId: peerId1 });
-    const { swarm: swarm2 } = await setupSwarm({ topic, peerId: peerId2 });
+    const peer1 = await setupSwarm({ topic });
+    const peer2 = await setupSwarm({ topic });
 
-    expect(swarm1.connections.length).toEqual(0);
-    expect(swarm2.connections.length).toEqual(0);
+    expect(peer1.swarm.connections.length).to.equal(0);
+    expect(peer2.swarm.connections.length).to.equal(0);
 
-    const connectPromises = Promise.all([swarm1.connected.waitForCount(1), swarm2.connected.waitForCount(1)]);
+    expect(peer1.swarm.connections.length).to.equal(0);
+    expect(peer2.swarm.connections.length).to.equal(0);
 
-    swarm1.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId2.asUint8Array(),
-        since: new Date(),
-      },
-    });
+    await connectSwarms(peer1, peer2, () => sleep(15));
+  }).timeout(10_000);
 
-    swarm2.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId1.asUint8Array(),
-        since: new Date(),
-      },
-    });
+  test('connection limiter', async () => {
+    // remotePeer1 <--> peer (connectionLimiter: max = 1) <--> remotePeer2
 
-    await connectPromises;
-  }).timeout(5_000);
+    const peerId = PublicKey.fromHex('7701dc2d');
+    const remotePeerId1 = PublicKey.fromHex('7d2bc6aa');
+    const remotePeerId2 = PublicKey.fromHex('39ba0e41');
 
-  test('second peer discovered after delay', async () => {
     const topic = PublicKey.random();
-    const peerId1 = PublicKey.random();
-    const peerId2 = PublicKey.random();
+    const connectionLimiter = new ConnectionLimiter({ maxConcurrentInitConnections: 1 });
 
-    const { swarm: swarm1, protocol: protocol1 } = await setupSwarm({ topic, peerId: peerId1 });
-
-    const { swarm: swarm2 } = await setupSwarm({ topic, peerId: peerId2 });
+    const signalManager = new MemorySignalManager(context);
+    const sendOriginal = signalManager.sendMessage.bind(signalManager);
+    const messages = new ComplexSet<{ author: PublicKey; recipient: PublicKey }>(
+      ({ author, recipient }) => author.toHex() + recipient.toHex(),
+    );
+    signalManager.sendMessage = async (message) => {
+      messages.add({ author: message.author, recipient: message.recipient });
+      return sendOriginal(message);
+    };
+    // Stop signaling to stop connection in initiation state.
+    signalManager.freeze();
+
+    const peer = await setupSwarm({
+      topic,
+      peerId,
+      connectionLimiter,
+      signalManager,
+    });
+    const remotePeer1 = await setupSwarm({ peerId: remotePeerId1, topic });
+    const remotePeer2 = await setupSwarm({ peerId: remotePeerId2, topic });
+
+    let connected1: Promise<void>;
+    {
+      // Connection limiter allow only one connection to be started.
+      const connectionInit = peer.swarm.connectionAdded.waitForCount(1);
+      connected1 = connectSwarms(peer, remotePeer1);
+      await asyncTimeout(connectionInit, 1000);
+    }
+
+    let connected2: Promise<void>;
+    {
+      // Connection limiter should prevent second connection from being started (only created).
+      const connectionInit = peer.swarm.connectionAdded.waitForCount(1);
+      connected2 = connectSwarms(peer, remotePeer2);
+      await asyncTimeout(connectionInit, 1000);
+    }
+
+    {
+      // Peer sent messages only to first remote peer.
+      expect(messages.has({ author: peer.peerId, recipient: remotePeer1.peerId })).to.equal(true);
+      expect(messages.has({ author: peer.peerId, recipient: remotePeer2.peerId })).to.equal(false);
+    }
+
+    signalManager.unfreeze();
+
+    await Promise.all([connected1, connected2]);
+  });
+});
 
-    expect(swarm1.connections.length).toEqual(0);
-    expect(swarm2.connections.length).toEqual(0);
+const connectSwarms = async (peer1: TestPeer, peer2: TestPeer, delay = async () => {}) => {
+  const connect1 = peer1.swarm.connected.waitForCount(1);
+  const connect2 = peer2.swarm.connected.waitForCount(1);
 
-    const connectPromises = Promise.all([swarm1.connected.waitForCount(1), swarm2.connected.waitForCount(1)]);
+  peer1.swarm.onSwarmEvent({
+    peerAvailable: {
+      peer: peer2.peerId.asUint8Array(),
+      since: new Date(),
+    },
+  });
 
-    swarm1.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId2.asUint8Array(),
-        since: new Date(),
-      },
-    });
-    await sleep(15);
-    swarm2.onSwarmEvent({
-      peerAvailable: {
-        peer: peerId1.asUint8Array(),
-        since: new Date(),
-      },
-    });
+  await delay();
 
-    await connectPromises;
+  peer2.swarm.onSwarmEvent({
+    peerAvailable: {
+      peer: peer1.peerId.asUint8Array(),
+      since: new Date(),
+    },
+  });
 
-    await protocol1.testConnection(peerId2);
-  }).timeout(10_000);
-});
+  if (
+    !(
+      peer1.swarm.connections.find((connection) => connection.remoteId.equals(peer2.peerId))?.state ===
+      ConnectionState.CONNECTED
+    )
+  ) {
+    await asyncTimeout(connect1, 3000);
+  }
+  if (
+    !(
+      peer2.swarm.connections.find((connection) => connection.remoteId.equals(peer1.peerId))?.state ===
+      ConnectionState.CONNECTED
+    )
+  ) {
+    await asyncTimeout(connect2, 3000);
+  }
+
+  await peer1.protocol.testConnection(peer2.peerId, 'test message 1');
+  await peer2.protocol.testConnection(peer1.peerId, 'test message 2');
+};

package/src/swarm/swarm.ts

@@ -2,11 +2,10 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, scheduleTask, sleep, synchronized } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import { ListeningHandle, Messenger } from '@dxos/messaging';
@@ -21,6 +20,7 @@ import { TransportFactory } from '../transport';
 import { Topic } from '../types';
 import { WireProtocolProvider } from '../wire-protocol';
 import { Connection, ConnectionState } from './connection';
+import { ConnectionLimiter } from './connection-limiter';
 import { Peer } from './peer';
 
 const INITIATION_DELAY = 100;
@@ -82,6 +82,7 @@ export class Swarm {
     private readonly _messenger: Messenger,
     private readonly _transportFactory: TransportFactory,
     private readonly _label: string | undefined,
+    private readonly _connectionLimiter: ConnectionLimiter,
   ) {
     log.trace(
       'dxos.mesh.swarm.constructor',
@@ -217,7 +218,6 @@ export class Swarm {
     return answer;
   }
 
-  @synchronized
   async onSignal(message: SignalMessage): Promise<void> {
     log('signal', { message });
     if (this._ctx.disposed) {
@@ -258,6 +258,7 @@ export class Swarm {
         this._swarmMessenger,
         this._protocolProvider,
         this._transportFactory,
+        this._connectionLimiter,
         {
           onInitiated: (connection) => {
             this.connectionAdded.emit(connection);

package/src/topology/fully-connected-topology.ts

@@ -2,8 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 
 import { SwarmController, Topology } from './topology';

package/src/topology/mmst-topology.ts

@@ -2,9 +2,9 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
 import distance from 'xor-distance';
 
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 

package/src/topology/star-topology.ts

@@ -2,8 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 

package/src/transport/memory-transport.ts

@@ -3,10 +3,10 @@
 //
 
 import { Transform } from 'node:stream';
-import invariant from 'tiny-invariant';
 
 import { Event, Trigger } from '@dxos/async';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import { Signal } from '@dxos/protocols/proto/dxos/mesh/swarm';

package/src/transport/webrtc-transport-proxy.ts

@@ -2,12 +2,13 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
+import { Writable } from 'node:stream';
 
 import { Event } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { ConnectionState, BridgeEvent, BridgeService } from '@dxos/protocols/proto/dxos/mesh/bridge';
@@ -57,18 +58,19 @@ export class WebRTCTransportProxy implements Transport {
           }
         });
 
-        const dataListener = async (data: Uint8Array) => {
-          try {
-            await this._params.bridgeService.sendData({
+        this._params.stream.pipe(new Writable({
+          write: (chunk, _, callback) => {
+            this._params.bridgeService.sendData({
               proxyId: this._proxyId,
-              payload: data
-            });
-          } catch (err: any) {
-            log.catch(err);
-          }
-        };
-        this._params.stream.on('data', dataListener);
-        this._ctx.onDispose(() => { this._params.stream.off('data', dataListener); });
+              payload: chunk
+            }).then(
+              () => callback(),
+              (err: any) => {
+                log.catch(err);
+              }
+            );
+          },
+        }));
       },
       (error) => log.catch(error)
     );

package/src/transport/webrtc-transport-service.ts

@@ -3,9 +3,9 @@
 //
 
 import { Duplex } from 'node:stream';
-import invariant from 'tiny-invariant';
 
 import { Stream } from '@dxos/codec-protobuf';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
@@ -21,20 +21,30 @@ import { ComplexMap } from '@dxos/util';
 
 import { WebRTCTransport } from './webrtc-transport';
 
+type TransportState = {
+  transport: WebRTCTransport;
+  stream: Duplex;
+  writeCallbacks: (() => void)[];
+};
+
 export class WebRTCTransportService implements BridgeService {
-  private readonly transports = new ComplexMap<PublicKey, { transport: WebRTCTransport; stream: Duplex }>(
-    PublicKey.hash,
-  );
+  private readonly transports = new ComplexMap<PublicKey, TransportState>(PublicKey.hash);
 
   // prettier-ignore
   constructor(
     private readonly _webrtcConfig?: any
-  ) {}
+  ) { }
 
   open(request: ConnectionRequest): Stream<BridgeEvent> {
     const rpcStream: Stream<BridgeEvent> = new Stream(({ ready, next, close }) => {
       const duplex: Duplex = new Duplex({
-        read: () => {},
+        read: () => {
+          const callbacks = [...transportState.writeCallbacks];
+          transportState.writeCallbacks.length = 0;
+          for (const cb of callbacks) {
+            cb();
+          }
+        },
         write: function (chunk, _, callback) {
           next({ data: { payload: chunk } });
           callback();
@@ -85,8 +95,15 @@ export class WebRTCTransportService implements BridgeService {
         close();
       });
 
+      const transportState: TransportState = {
+        transport,
+        stream: duplex,
+        writeCallbacks: [],
+      };
+
       ready();
-      this.transports.set(request.proxyId, { transport, stream: duplex });
+
+      this.transports.set(request.proxyId, transportState);
     });
 
     return rpcStream;
@@ -99,7 +116,13 @@ export class WebRTCTransportService implements BridgeService {
 
   async sendData({ proxyId, payload }: DataRequest): Promise<void> {
     invariant(this.transports.has(proxyId));
-    await this.transports.get(proxyId)!.stream.push(payload);
+    const state = this.transports.get(proxyId)!;
+    const bufferHasSpace = state.stream.push(payload);
+    if (!bufferHasSpace) {
+      await new Promise<void>((resolve) => {
+        state.writeCallbacks.push(resolve);
+      });
+    }
   }
 
   async close({ proxyId }: CloseRequest) {

package/src/transport/webrtc-transport.ts

@@ -3,10 +3,10 @@
 //
 
 import SimplePeerConstructor, { Instance as SimplePeer } from 'simple-peer';
-import invariant from 'tiny-invariant';
 
 import { Event } from '@dxos/async';
 import { ErrorStream, raise } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/protocols';