@dxos/network-manager 0.1.9 → 0.1.11

package/src/testing/test-builder.ts

@@ -4,96 +4,176 @@
 
 import { PublicKey } from '@dxos/keys';
 import { MemorySignalManager, MemorySignalManagerContext, WebsocketSignalManager } from '@dxos/messaging';
-import { afterTest } from '@dxos/testutils';
+import { schema } from '@dxos/protocols';
+import { createLinkedPorts, createProtoRpcPeer, ProtoRpcPeer } from '@dxos/rpc';
+import { ComplexMap } from '@dxos/util';
 
 import { NetworkManager } from '../network-manager';
-import { FullyConnectedTopology, Topology } from '../topology';
-import { MemoryTransportFactory, TransportFactory } from '../transport';
+import { FullyConnectedTopology } from '../topology';
+import {
+  MemoryTransportFactory,
+  TransportFactory,
+  WebRTCTransport,
+  WebRTCTransportProxyFactory,
+  WebRTCTransportService
+} from '../transport';
 import { TestProtocolPlugin, testProtocolProvider } from './test-protocol';
 
 // Signal server will be started by the setup script.
 export const TEST_SIGNAL_URL = 'ws://localhost:4000/.well-known/dx/signal';
 
 export type TestBuilderOptions = {
-  signalUrl?: string;
+  signalHosts?: string[];
+  bridge?: boolean;
 };
 
-// TODO(burdon): Builder pattern (see feed-store).
+/**
+ * Builder used to construct networks and peers.
+ */
 export class TestBuilder {
   private _signalContext = new MemorySignalManagerContext();
 
-  constructor(private _options: TestBuilderOptions = {}) {}
+  constructor(public readonly options: TestBuilderOptions = {}) {}
 
   createSignalManager() {
-    if (this._options.signalUrl) {
-      return new WebsocketSignalManager([this._options.signalUrl]);
+    if (this.options.signalHosts) {
+      return new WebsocketSignalManager(this.options.signalHosts);
     }
 
     return new MemorySignalManager(this._signalContext);
   }
 
+  createPeer() {
+    return new TestPeer(this);
+  }
+}
+
+/**
+ * Testing network peer.
+ */
+export class TestPeer {
+  readonly peerId = PublicKey.random();
+
+  private readonly _swarms = new ComplexMap<PublicKey, TestSwarmConnection>(PublicKey.hash);
+
+  /**
+   * @internal
+   */
+  readonly _networkManager: NetworkManager;
+
+  private _proxy?: ProtoRpcPeer<any>;
+  private _service?: ProtoRpcPeer<any>;
+
+  constructor(private readonly testBuilder: TestBuilder) {
+    this._networkManager = this.createNetworkManager();
+  }
+
+  // TODO(burdon): Move to TestBuilder.
   createNetworkManager() {
+    let transportFactory: TransportFactory = MemoryTransportFactory;
+
+    if (this.testBuilder.options.signalHosts) {
+      if (this.testBuilder.options.bridge) {
+        // Simulates bridge to shared worker.
+        const [proxyPort, servicePort] = createLinkedPorts();
+
+        this._proxy = createProtoRpcPeer({
+          port: proxyPort,
+          requested: {
+            BridgeService: schema.getService('dxos.mesh.bridge.BridgeService')
+          },
+          noHandshake: true,
+          encodingOptions: {
+            preserveAny: true
+          }
+        });
+
+        this._service = createProtoRpcPeer({
+          port: servicePort,
+          exposed: {
+            BridgeService: schema.getService('dxos.mesh.bridge.BridgeService')
+          },
+          handlers: { BridgeService: new WebRTCTransportService() },
+          noHandshake: true,
+          encodingOptions: {
+            preserveAny: true
+          }
+        });
+
+        transportFactory = new WebRTCTransportProxyFactory().setBridgeService(this._proxy.rpc.BridgeService);
+      } else {
+        transportFactory = {
+          createTransport: (params) => new WebRTCTransport(params)
+        };
+      }
+    }
+
     return new NetworkManager({
-      signalManager: new MemorySignalManager(this._signalContext),
-      transportFactory: MemoryTransportFactory
+      signalManager: this.testBuilder.createSignalManager(),
+      transportFactory
     });
   }
-}
 
-export interface CreatePeerOptions {
-  topic: PublicKey;
-  peerId: PublicKey;
-  topology?: Topology;
-  signalHosts?: string[];
-  transportFactory: TransportFactory;
+  async open() {
+    await this._proxy?.open();
+    await this._service?.open();
+  }
+
+  async close() {
+    await Promise.all(Array.from(this._swarms.values()).map((swarm) => swarm.leave()));
+    this._swarms.clear();
+
+    await this._proxy?.close();
+    await this._service?.close();
+    await this._networkManager.close();
+  }
+
+  getSwarm(topic: PublicKey): TestSwarmConnection {
+    const swarm = this._swarms.get(topic);
+    if (!swarm) {
+      throw new Error(`Swarm not found for topic: ${topic}`);
+    }
+
+    return swarm;
+  }
+
+  createSwarm(topic: PublicKey): TestSwarmConnection {
+    // TODO(burdon): Multiple.
+    // if (this._swarms.get(topic)) {
+    //   throw new Error(`Swarm already exists for topic: ${topic.truncate()}`);
+    // }
+
+    const swarm = new TestSwarmConnection(this, topic);
+    this._swarms.set(topic, swarm);
+    return swarm;
+  }
 }
 
-// TODO(burdon): Remove.
-const signalContext = new MemorySignalManagerContext();
+// TODO(burdon): Reconcile with new Swarm concept.
+export class TestSwarmConnection {
+  plugin: TestProtocolPlugin;
 
-/**
- * @deprecated
- */
-export const createPeer = async ({
-  topic,
-  peerId,
-  topology = new FullyConnectedTopology(),
-  signalHosts,
-  transportFactory
-}: CreatePeerOptions) => {
-  const signalManager = signalHosts ? new WebsocketSignalManager(signalHosts!) : new MemorySignalManager(signalContext);
-  await signalManager.subscribeMessages(peerId);
-  const networkManager = new NetworkManager({
-    signalManager,
-    transportFactory
-  });
-
-  afterTest(() => networkManager.destroy());
-
-  // TODO(burdon): Use keys everywhere.
-  const plugin = new TestProtocolPlugin(peerId.asBuffer());
-  const protocolProvider = testProtocolProvider(topic.asBuffer(), peerId, plugin);
-  await networkManager.openSwarmConnection({ topic, peerId, protocol: protocolProvider, topology });
-
-  return {
-    networkManager,
-    plugin
-  };
-};
+  constructor(readonly peer: TestPeer, readonly topic: PublicKey) {
+    // TODO(burdon): Configure plugins.
+    // TODO(burdon): Prevent reuse?
+    this.plugin = new TestProtocolPlugin(this.peer.peerId.asBuffer()); // TODO(burdon): PublicKey.
+  }
 
-// const createMemoryNetworkManager = (signalContext: MemorySignalManagerContext) => {
-//   return new NetworkManager({
-//     signalManager: new MemorySignalManager(signalContext),
-//     transportFactory: MemoryTransportFactory
-//   });
-// };
+  // TODO(burdon): Need to create new plugin instance per swarm?
+  //  If so, then perhaps joinSwarm should return swarm object with access to plugins.
+  async join(topology = new FullyConnectedTopology()) {
+    await this.peer._networkManager.joinSwarm({
+      topic: this.topic,
+      peerId: this.peer.peerId,
+      protocol: testProtocolProvider(this.topic.asBuffer(), this.peer.peerId, this.plugin),
+      topology
+    });
 
-// const createWebRTCNetworkManager = async (config: any, peerId: PublicKey) => {
-//   const signalManager = new WebsocketSignalManager(signalHosts!);
-//   await signalManager.subscribeMessages(peerId);
-//
-//   return new NetworkManager({
-//     signalManager,
-//     transportFactory: createWebRTCTransportFactory(config)
-//   });
-// };
+    return this;
+  }
+
+  async leave() {
+    await this.peer._networkManager.leaveSwarm(this.topic);
+    return this;
+  }
+}

package/src/testing/test-protocol.ts

@@ -15,15 +15,31 @@ import { protocolFactory } from '../protocol-factory';
 
 const EXTENSION_NAME = 'test';
 
+// TODO(burdon): PublicKey.
 // TODO(dboreham): This method should be added to Protocol (and one for "my ID"?).
 export const getPeerId = (protocol: Protocol) => {
   const { peerId } = protocol.getSession() ?? {};
   if (!peerId) {
     return undefined;
   }
+
   return PublicKey.bufferize(peerId);
 };
 
+/**
+ * @deprecated
+ */
+// TODO(burdon): Reconcile with transportProtocolProvider.
+export const testProtocolProvider = (topic: Buffer, peerId: PublicKey, protocolPlugin: any) => {
+  log('creating protocol-factory', { topic: PublicKey.from(topic), peerId });
+
+  return protocolFactory({
+    getTopics: () => [topic],
+    session: { peerId: peerId.toHex() }, // TODO(burdon): PublicKey.
+    plugins: [protocolPlugin]
+  });
+};
+
 /**
  * Object Implementing a p2p protocol (interaction across a set of peers using a common protocol)
  * and an associated transport connection protocol (interaction with each individual peer over a single network
@@ -81,9 +97,13 @@ export class TestProtocolPlugin extends EventEmitter {
       .setCloseHandler(this._onPeerDisconnect.bind(this));
   }
 
-  // Methods below are per-peer-connection.
+  // Methods below are per-peer connection.
   // TODO(dboreham): Why are these here and not on Protocol?
 
+  private async _init(protocol: Protocol) {
+    this.initCalled = true;
+  }
+
   /**
    * Send/Receive messages with peer when initiating a request/response interaction.
    * @param peerId Must be the value passed to the constructor on the responding node.
@@ -92,38 +112,38 @@ export class TestProtocolPlugin extends EventEmitter {
    */
   async send(peerId: Buffer, payload: string): Promise<string> {
     assert(Buffer.isBuffer(peerId));
-    const peerIdStr = PublicKey.stringify(peerId);
+    const peerIdStr = PublicKey.stringify(peerId); // TODO(burdon): PublicKey.
     const peer = this._peers.get(peerIdStr);
-    // TODO(dboreham): Throw fatal error if peer not found.
+    if (!peer) {
+      throw new Error(`peer not found: ${PublicKey.from(peerId).truncate()}`);
+    }
+
     const extension = peer.getExtension(EXTENSION_NAME);
     const encoded = Buffer.from(payload);
 
-    log('sent', { peerIdStr, payload });
+    log('sending', { peerId: PublicKey.from(peerId), payload });
     return await extension.send(encoded, { oneway: true });
   }
 
-  private async _init(protocol: Protocol) {
-    this.initCalled = true;
-  }
-
   async _receive(protocol: Protocol, data: any) {
     const peerId = getPeerId(protocol);
     assert(peerId !== undefined);
-    const peerIdStr = PublicKey.stringify(peerId);
     let payload = data.data.toString();
     if (this._uppercase) {
       payload = payload.toUpperCase();
     }
 
-    log('received', { peerIdStr, payload });
+    log('received', { peerId: PublicKey.from(peerId), payload });
     this.emit('receive', protocol, payload);
   }
 
   async _onPeerConnect(protocol: Protocol) {
+    log('connecting', { id: PublicKey.from(protocol.id) });
     const peerId = getPeerId(protocol);
     if (peerId === undefined) {
       return;
     }
+
     const peerIdStr = PublicKey.stringify(peerId);
     if (this._peers.has(peerIdStr)) {
       return;
@@ -134,6 +154,7 @@ export class TestProtocolPlugin extends EventEmitter {
   }
 
   async _onPeerDisconnect(protocol: Protocol) {
+    log('disconnecting', { id: PublicKey.from(protocol.id) });
     const peerId = getPeerId(protocol);
     if (peerId === undefined) {
       return;
@@ -143,16 +164,3 @@ export class TestProtocolPlugin extends EventEmitter {
     this.emit('disconnect', peerId);
   }
 }
-
-/**
- * @return {ProtocolProvider}
- */
-// TODO(dboreham): Try to encapsulate swarm_key, nodeId.
-export const testProtocolProvider = (swarmKey: Buffer, peerId: PublicKey, protocolPlugin: any) => {
-  log('creating protocol factory', { swarmKey: PublicKey.from(swarmKey), peerId });
-  return protocolFactory({
-    getTopics: () => [swarmKey],
-    session: { peerId: peerId.toHex() },
-    plugins: [protocolPlugin]
-  });
-};

package/src/tests/basic-test-suite.ts

@@ -0,0 +1,168 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { expect } from 'chai';
+import waitForExpect from 'wait-for-expect';
+
+import { latch } from '@dxos/async';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Protocol } from '@dxos/mesh-protocol';
+import { range } from '@dxos/util';
+
+import { TestBuilder } from '../testing';
+import { FullyConnectedTopology, StarTopology } from '../topology';
+import { exchangeMessages, joinSwarm, leaveSwarm, openAndCloseAfterTest } from './utils';
+
+// TODO(burdon): Use PublicKey throughout (remove conversion to strings, from buffers, etc.)
+
+/**
+ * Basic swarm tests.
+ */
+export const basicTestSuite = (testBuilder: TestBuilder, runTests = true) => {
+  if (runTests) {
+    return;
+  }
+
+  it('joins swarm, sends messages, and cleanly exits', async () => {
+    const peer1 = testBuilder.createPeer();
+    const peer2 = testBuilder.createPeer();
+    await openAndCloseAfterTest([peer1, peer2]);
+
+    const topic = PublicKey.random();
+    const [swarm1, swarm2] = await joinSwarm([peer1, peer2], topic, () => new FullyConnectedTopology());
+    await exchangeMessages(swarm1, swarm2);
+    await leaveSwarm([peer1, peer2], topic);
+  });
+
+  // TODO(burdon): Test with more peers (configure and test messaging).
+  it('joins swarm with star topology', async () => {
+    const peer1 = testBuilder.createPeer();
+    const peer2 = testBuilder.createPeer();
+    await openAndCloseAfterTest([peer1, peer2]);
+
+    const topic = PublicKey.random();
+    const [swarm1, swarm2] = await joinSwarm([peer1, peer2], topic, () => new StarTopology(peer1.peerId)); // NOTE: Same peer.
+    await exchangeMessages(swarm1, swarm2);
+    await leaveSwarm([peer1, peer2], topic);
+  });
+
+  // TODO(burdon): Fails when trying to reconnect to same topic.
+  it('joins swarm multiple times', async () => {
+    const peer1 = testBuilder.createPeer();
+    const peer2 = testBuilder.createPeer();
+    await openAndCloseAfterTest([peer1, peer2]);
+
+    const topic1 = PublicKey.random();
+
+    {
+      const [swarm1, swarm2] = await joinSwarm([peer1, peer2], topic1);
+      await exchangeMessages(swarm1, swarm2);
+      await leaveSwarm([peer1, peer2], topic1);
+    }
+
+    // TODO(burdon): Add log marker like this to logging lib. Auto add between tests?
+    // TODO(burdon): Enable new Error to take second context obj: new Error('msg', {}).
+    log('————————————————————————————————————————');
+
+    const topic2 = PublicKey.random();
+
+    {
+      const [swarm1, swarm2] = await joinSwarm([peer1, peer2], topic2);
+      await exchangeMessages(swarm1, swarm2);
+      await leaveSwarm([peer1, peer2], topic2);
+    }
+  });
+
+  it.skip('joins multiple swarms', async () => {
+    // TODO(burdon): N peers.
+    // TODO(burdon): Merge with test below.
+    const peer1 = testBuilder.createPeer();
+    const peer2 = testBuilder.createPeer();
+    await openAndCloseAfterTest([peer1, peer2]);
+
+    const numSwarms = 5;
+    const topics = Array.from(Array(numSwarms)).map(() => PublicKey.random());
+    expect(topics).to.have.length(numSwarms);
+  });
+
+  // TODO(burdon): Fails with WebRTC.
+  // TODO(burdon): This just tests multiple swarms (not peers in the same swarm)?
+  //  Generalize for n swarms and factor into other tests; configure message activity.
+  it.skip('joins multiple swarms concurrently', async () => {
+    const createSwarm = async (messages: any[], label: string) => {
+      const topicA = PublicKey.random();
+      const peer1a = testBuilder.createPeer();
+      const peer2a = testBuilder.createPeer();
+
+      const swarm1a = await peer1a.createSwarm(topicA).join();
+      const swarm2a = await peer2a.createSwarm(topicA).join();
+
+      swarm1a.plugin.on('receive', (_, msg: string) => {
+        messages.push(msg);
+      });
+      swarm2a.plugin.on('connect', async () => {
+        await swarm2a.plugin.send(peer1a.peerId.asBuffer(), label);
+      });
+    };
+
+    const receivedA: any[] = [];
+    const receivedB: any[] = [];
+
+    void createSwarm(receivedA, 'Swarm A');
+    void createSwarm(receivedB, 'Swarm B');
+
+    // TODO(burdon): Replace with triggers.
+    await waitForExpect(() => {
+      expect(receivedA).to.have.length(1);
+      expect(receivedB).to.have.length(1);
+
+      expect(receivedA[0]).to.eq('Swarm A');
+      expect(receivedB[0]).to.eq('Swarm B');
+    });
+  });
+
+  // TODO(burdon): Fails with WebRTC.
+  // TODO(burdon): Factor out components of test.
+  it.skip('many peers and connections', async () => {
+    const numTopics = 5;
+    const peersPerTopic = 5;
+
+    await Promise.all(
+      range(numTopics).map(async () => {
+        const topic = PublicKey.random();
+
+        await Promise.all(
+          range(peersPerTopic).map(async () => {
+            const peer = testBuilder.createPeer();
+            const swarm = await peer.createSwarm(topic).join();
+
+            const [done, pongReceived] = latch({ count: peersPerTopic - 1 });
+
+            swarm.plugin.on('connect', async (protocol: Protocol) => {
+              const { peerId } = protocol.getSession() ?? {};
+              const remoteId = PublicKey.from(peerId);
+              await swarm.plugin.send(remoteId.asBuffer(), 'ping');
+            });
+
+            swarm.plugin.on('receive', async (protocol: Protocol, data: any) => {
+              const { peerId } = protocol.getSession() ?? {};
+              const remoteId = PublicKey.from(peerId);
+
+              if (data === 'ping') {
+                await swarm.plugin.send(remoteId.asBuffer(), 'pong');
+              } else if (data === 'pong') {
+                pongReceived();
+              } else {
+                throw new Error(`Invalid message: ${data}`);
+              }
+            });
+
+            await done();
+          })
+        );
+      })
+    );
+  });
+};

package/src/tests/memory-transport.test.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import { TestBuilder } from '../testing';
+import { basicTestSuite } from './basic-test-suite';
+
+describe('Memory transport', function () {
+  const testBuilder = new TestBuilder();
+  basicTestSuite(testBuilder);
+});

package/src/{presence.test.ts → tests/presence.test.ts}

@@ -2,7 +2,7 @@
 // Copyright 2021 DXOS.org
 //
 
-import expect from 'expect';
+import { expect } from 'chai';
 import waitForExpect from 'wait-for-expect';
 
 import { PublicKey } from '@dxos/keys';
@@ -10,13 +10,14 @@ import { MemorySignalManagerContext, MemorySignalManager } from '@dxos/messaging
 import { PresencePlugin } from '@dxos/protocol-plugin-presence';
 import { afterTest } from '@dxos/testutils';
 
-import { NetworkManager } from './network-manager';
-import { createProtocolFactory } from './protocol-factory';
-import { FullyConnectedTopology } from './topology';
-import { MemoryTransportFactory } from './transport';
+import { NetworkManager } from '../network-manager';
+import { createProtocolFactory } from '../protocol-factory';
+import { FullyConnectedTopology } from '../topology';
+import { MemoryTransportFactory } from '../transport';
 
 const signalContext = new MemorySignalManagerContext();
 
+// TODO(burdon): Move to TestBuilder (configure plugins).
 const createPeer = async (topic: PublicKey) => {
   const peerId = PublicKey.random();
 
@@ -24,12 +25,12 @@ const createPeer = async (topic: PublicKey) => {
     signalManager: new MemorySignalManager(signalContext),
     transportFactory: MemoryTransportFactory
   });
-  afterTest(() => networkManager.destroy());
+  afterTest(() => networkManager.close());
 
   const presencePlugin = new PresencePlugin(peerId.asBuffer());
   afterTest(() => presencePlugin.stop());
 
-  await networkManager.openSwarmConnection({
+  await networkManager.joinSwarm({
     peerId,
     protocol: createProtocolFactory(topic, peerId, [presencePlugin]),
     topic,
@@ -40,38 +41,38 @@ const createPeer = async (topic: PublicKey) => {
 };
 
 describe('Presence', function () {
-  it('sees connected peers', async function () {
+  it('detects connected peers', async function () {
+    // TODO(burdon): Configure plugins.
+    // const testBuilder = new TestBuilder();
+    // const peer1 = testBuilder.createPeer();
+    // const peer2 = testBuilder.createPeer();
+
     const topic = PublicKey.random();
     const peer1 = await createPeer(topic);
     const peer2 = await createPeer(topic);
+    // await peer1.joinSwarm(topic);
+    // await peer2.joinSwarm(topic);
 
+    // TODO(burdon): Use triggers instead of waitForExpect.
     await waitForExpect(() => {
-      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).to.deep.eq(
         [peer1, peer2].map((key) => key.peerId.toHex()).sort()
       );
 
-      expect(peer2.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+      expect(peer2.presence.peers.map((key) => key.toString('hex')).sort()).to.deep.eq(
         [peer1, peer2].map((key) => key.peerId.toHex()).sort()
       );
     });
-  });
 
-  it('removes disconnected peers', async function () {
-    const topic = PublicKey.random();
-    const peer1 = await createPeer(topic);
-    const peer2 = await createPeer(topic);
+    await peer2.networkManager.leaveSwarm(topic);
 
     await waitForExpect(() => {
-      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
-        [peer1, peer2].map((key) => key.peerId.toHex()).sort()
+      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).to.deep.eq(
+        [peer1].map((key) => key.peerId.toHex()).sort()
       );
-    });
 
-    await peer2.networkManager.closeSwarmConnection(topic);
-
-    await waitForExpect(() => {
-      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
-        [peer1].map((x) => x.peerId.toHex()).sort()
+      expect(peer2.presence.peers.map((key) => key.toString('hex')).sort()).to.deep.eq(
+        [peer2].map((key) => key.peerId.toHex()).sort()
       );
     });
   });