@dxos/network-manager 0.1.0

package/src/network-manager.ts

@@ -0,0 +1,204 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { Event } from '@dxos/async';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Protocol } from '@dxos/mesh-protocol';
+import { Messenger, SignalManager } from '@dxos/messaging';
+import { ComplexMap } from '@dxos/util';
+
+import { ConnectionLog } from './connection-log';
+import { SignalConnection } from './signal';
+import { Swarm, SwarmMapper } from './swarm';
+import { Topology } from './topology';
+import { TransportFactory } from './transport';
+
+export type ProtocolProvider = (opts: { channel: Buffer; initiator: boolean }) => Protocol;
+
+export type SwarmConnection = {
+  close(): Promise<void>;
+};
+
+// TODO(burdon): Timeout.
+export type SwarmOptions = {
+  /**
+   * Swarm topic.
+   */
+  topic: PublicKey;
+
+  /**
+   * This node's peer id.
+   */
+  peerId: PublicKey;
+
+  /**
+   * Protocol to use for every connection.
+   */
+  protocol: ProtocolProvider;
+
+  /**
+   * Requested topology. Must be a new instance for every swarm.
+   */
+  topology?: Topology;
+
+  /**
+   * Presence plugin for network mapping, if exists.
+   */
+  presence?: any;
+
+  /**
+   * Custom label assigned to this swarm. Used in devtools to display human-readable names for swarms.
+   */
+  label?: string;
+};
+
+export type NetworkManagerOptions = {
+  transportFactory: TransportFactory;
+  signalManager: SignalManager;
+  log?: boolean; // TODO(burdon): Remove.
+};
+
+/**
+ * Manages connection to the swarm.
+ */
+export class NetworkManager {
+  private readonly _transportFactory: TransportFactory;
+
+  private readonly _swarms = new ComplexMap<PublicKey, Swarm>(PublicKey.hash);
+  private readonly _mappers = new ComplexMap<PublicKey, SwarmMapper>(PublicKey.hash);
+
+  private readonly _signalManager: SignalManager;
+  private readonly _messenger: Messenger;
+  private readonly _signalConnection: SignalConnection;
+  private readonly _connectionLog?: ConnectionLog;
+
+  public readonly topicsUpdated = new Event<void>();
+
+  constructor({ transportFactory, signalManager, log }: NetworkManagerOptions) {
+    this._transportFactory = transportFactory;
+
+    // Listen for signal manager events.
+    this._signalManager = signalManager;
+    this._signalManager.swarmEvent.on(({ topic, swarmEvent: event }) => this._swarms.get(topic)?.onSwarmEvent(event));
+    this._messenger = new Messenger({ signalManager: this._signalManager });
+
+    this._signalConnection = {
+      join: (opts) => this._signalManager.join(opts),
+      leave: (opts) => this._signalManager.leave(opts)
+    };
+
+    if (log) {
+      this._connectionLog = new ConnectionLog();
+    }
+  }
+
+  get signal() {
+    return this._signalManager;
+  }
+
+  // TODO(burdon): Reconcile with "discovery_key".
+  get topics() {
+    return Array.from(this._swarms.keys());
+  }
+
+  // TODO(burdon): Factor out devtools.
+  get connectionLog() {
+    return this._connectionLog;
+  }
+
+  getSwarmMap(topic: PublicKey): SwarmMapper | undefined {
+    return this._mappers.get(topic);
+  }
+
+  getSwarm(topic: PublicKey): Swarm | undefined {
+    return this._swarms.get(topic);
+  }
+
+  /**
+   * Join the swarm.
+   */
+  // TODO(burdon): Join/Open?
+  async openSwarmConnection({
+    topic,
+    peerId,
+    topology,
+    protocol,
+    presence,
+    label
+  }: SwarmOptions): Promise<SwarmConnection> {
+    assert(PublicKey.isPublicKey(topic));
+    assert(PublicKey.isPublicKey(peerId));
+    assert(topology);
+    assert(typeof protocol === 'function');
+    if (this._swarms.has(topic)) {
+      throw new Error(`Already connected to swarm: ${topic}`);
+    }
+
+    log('joining', { topic, peerId, topology: topology.toString() });
+    const swarm = new Swarm(topic, peerId, topology, protocol, this._messenger, this._transportFactory, label);
+    swarm.errors.handle((error) => {
+      log(`Swarm error: ${error}`);
+    });
+
+    this._swarms.set(topic, swarm);
+    this._signalConnection.join({ topic, peerId }).catch((error) => log.catch(error));
+    this._mappers.set(topic, new SwarmMapper(swarm, presence));
+
+    this.topicsUpdated.emit();
+    this._connectionLog?.swarmJoined(swarm);
+    log('open', { topic });
+
+    return {
+      close: () => this.closeSwarmConnection(topic)
+    };
+  }
+
+  /**
+   * Close the connection.
+   */
+  async closeSwarmConnection(topic: PublicKey) {
+    if (!this._swarms.has(topic)) {
+      return;
+    }
+
+    log('closing', { topic });
+    const swarm = this._swarms.get(topic)!;
+    await this._signalConnection.leave({ topic, peerId: swarm.ownPeerId });
+
+    const map = this._mappers.get(topic)!;
+    map.destroy();
+    this._mappers.delete(topic);
+
+    this._connectionLog?.swarmLeft(swarm);
+
+    await swarm.destroy();
+    this._swarms.delete(topic);
+
+    await this.topicsUpdated.emit();
+    log('closed', { topic });
+  }
+
+  /**
+   * @deprecated
+   */
+  // TODO(dmaretskyi): Remove.
+  async start() {
+    console.warn('NetworkManger.start is deprecated.');
+  }
+
+  // TODO(burdon): Open/close?
+  async destroy() {
+    for (const topic of this._swarms.keys()) {
+      await this.closeSwarmConnection(topic).catch((err) => {
+        log(`Failed to leave swarm ${topic} on NetworkManager.destroy}`);
+        log(err);
+      });
+    }
+
+    await this._signalManager.destroy();
+  }
+}

package/src/presence.test.ts

@@ -0,0 +1,78 @@
+//
+// Copyright 2021 DXOS.org
+//
+
+import expect from 'expect';
+import waitForExpect from 'wait-for-expect';
+
+import { PublicKey } from '@dxos/keys';
+import { MemorySignalManagerContext, MemorySignalManager } from '@dxos/messaging';
+import { PresencePlugin } from '@dxos/protocol-plugin-presence';
+import { afterTest } from '@dxos/testutils';
+
+import { NetworkManager } from './network-manager';
+import { createProtocolFactory } from './protocol-factory';
+import { FullyConnectedTopology } from './topology';
+import { MemoryTransportFactory } from './transport';
+
+const signalContext = new MemorySignalManagerContext();
+
+const createPeer = async (topic: PublicKey) => {
+  const peerId = PublicKey.random();
+
+  const networkManager = new NetworkManager({
+    signalManager: new MemorySignalManager(signalContext),
+    transportFactory: MemoryTransportFactory
+  });
+  afterTest(() => networkManager.destroy());
+
+  const presencePlugin = new PresencePlugin(peerId.asBuffer());
+  afterTest(() => presencePlugin.stop());
+
+  await networkManager.openSwarmConnection({
+    peerId,
+    protocol: createProtocolFactory(topic, peerId, [presencePlugin]),
+    topic,
+    topology: new FullyConnectedTopology()
+  });
+
+  return { peerId, presence: presencePlugin, networkManager };
+};
+
+describe('Presence', function () {
+  it('sees connected peers', async function () {
+    const topic = PublicKey.random();
+    const peer1 = await createPeer(topic);
+    const peer2 = await createPeer(topic);
+
+    await waitForExpect(() => {
+      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+        [peer1, peer2].map((key) => key.peerId.toHex()).sort()
+      );
+
+      expect(peer2.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+        [peer1, peer2].map((key) => key.peerId.toHex()).sort()
+      );
+    });
+  });
+
+  it('removes disconnected peers', async function () {
+    const topic = PublicKey.random();
+    const peer1 = await createPeer(topic);
+    const peer2 = await createPeer(topic);
+
+    await waitForExpect(() => {
+      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+        [peer1, peer2].map((key) => key.peerId.toHex()).sort()
+      );
+    });
+
+    await peer2.networkManager.closeSwarmConnection(topic);
+
+    await waitForExpect(() => {
+      expect(peer1.presence.peers.map((key) => key.toString('hex')).sort()).toEqual(
+        [peer1].map((x) => x.peerId.toHex()).sort()
+      );
+    });
+  });
+});

package/src/protocol-factory.ts

@@ -0,0 +1,83 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import assert from 'node:assert';
+
+import { discoveryKey } from '@dxos/crypto';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { Extension, Protocol } from '@dxos/mesh-protocol';
+
+import { ProtocolProvider } from './network-manager';
+
+interface ProtocolFactoryOptions {
+  plugins: any[];
+  getTopics: () => Buffer[];
+  session: Record<string, any>;
+}
+
+/**
+ * Returns a function that takes a channel parameter, returns a Protocol object
+ * with its context set to channel, plugins from plugins parameter and session
+ * set to session parameter.
+ *
+ * @param plugins array of Protocol plugin objects to add to created Protocol objects
+ * @param getTopics retrieve a list of known topic keys to encrypt by public key.
+ * @deprecated Use `createProtocolFactory`.
+ */
+export const protocolFactory = ({
+  session = {},
+  plugins = [],
+  getTopics
+}: ProtocolFactoryOptions): ProtocolProvider => {
+  assert(getTopics);
+  // eslint-disable-next-line no-unused-vars
+  return ({ channel, initiator }) => {
+    const protocol = new Protocol({
+      streamOptions: { live: true },
+      discoveryToPublicKey: (dk) => {
+        const publicKey = getTopics().find((topic) => discoveryKey(topic).equals(dk));
+        if (publicKey) {
+          protocol.setContext({ topic: PublicKey.stringify(publicKey) });
+        }
+        assert(publicKey, 'PublicKey not found in discovery.');
+        return publicKey;
+      },
+      initiator,
+      userSession: session,
+      discoveryKey: channel
+    });
+
+    protocol.setExtensions(plugins.map((plugin) => plugin.createExtension())).init();
+
+    log('Created protocol');
+    return protocol;
+  };
+};
+
+export interface Plugin {
+  createExtension: () => Extension;
+}
+
+export const createProtocolFactory = (topic: PublicKey, peerId: PublicKey, plugins: Plugin[]) =>
+  protocolFactory({
+    getTopics: () => [topic.asBuffer()],
+    session: { peerId: PublicKey.stringify(peerId.asBuffer()) },
+    plugins
+  });
+
+/**
+ * Creates a ProtocolProvider for simple transport connections with only one protocol plugin.
+ * @deprecated Use `createProtocolFactory`.
+ */
+export const transportProtocolProvider = (
+  rendezvousKey: Buffer,
+  peerId: Buffer,
+  protocolPlugin: any
+): ProtocolProvider =>
+  protocolFactory({
+    getTopics: () => [rendezvousKey],
+    session: { peerId: PublicKey.stringify(peerId) },
+    plugins: [protocolPlugin]
+  });

package/src/signal/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+export * from './signal-connection';
+export * from './signal-messaging';
+export * from './message-router';

package/src/signal/integration.test.ts

@@ -0,0 +1,129 @@
+//
+// Copyright 2022 DXOS.org
+//
+
+// @dxos/mocha platform=nodejs
+
+import { expect } from 'earljs';
+import waitForExpect from 'wait-for-expect';
+
+import { PublicKey } from '@dxos/keys';
+import { WebsocketSignalManager } from '@dxos/messaging';
+import { createTestBroker, TestBroker } from '@dxos/signal';
+
+import { MessageRouter } from './message-router';
+import { SignalMessage } from './signal-messaging';
+
+describe('Signal Integration Test', function () {
+  let broker: TestBroker;
+
+  before(async function () {
+    broker = await createTestBroker();
+  });
+
+  after(function () {
+    broker.stop();
+  });
+
+  const setupPeer = ({ topic = PublicKey.random() }: { topic?: PublicKey } = {}) => {
+    const signalManager = new WebsocketSignalManager([broker.url()]);
+    signalManager.onMessage.on((message) => messageRouter.receiveMessage(message));
+
+    const receivedSignals: SignalMessage[] = [];
+    const signalMock = async (msg: SignalMessage) => {
+      receivedSignals.push(msg);
+    };
+    const messageRouter = new MessageRouter({
+      sendMessage: signalManager.sendMessage.bind(signalManager),
+      onSignal: signalMock,
+      onOffer: async () => ({ accept: true }),
+      topic
+    });
+
+    return {
+      signalManager,
+      receivedSignals,
+      messageRouter
+    };
+  };
+
+  it('two peers connecting', async function () {
+    const peer1 = PublicKey.random();
+    const peer2 = PublicKey.random();
+    const topic = PublicKey.random();
+
+    const peerNetworking1 = setupPeer({ topic });
+    const peerNetworking2 = setupPeer({ topic });
+
+    const promise1 = peerNetworking1.signalManager.swarmEvent.waitFor(
+      ({ swarmEvent }) => !!swarmEvent.peerAvailable && peer2.equals(swarmEvent.peerAvailable.peer)
+    );
+    const promise2 = peerNetworking1.signalManager.swarmEvent.waitFor(
+      ({ swarmEvent }) => !!swarmEvent.peerAvailable && peer1.equals(swarmEvent.peerAvailable.peer)
+    );
+
+    await peerNetworking1.signalManager.join({ topic, peerId: peer1 });
+    await peerNetworking2.signalManager.join({ topic, peerId: peer2 });
+
+    await promise1;
+    await promise2;
+
+    expect(
+      await peerNetworking1.messageRouter.offer({
+        topic,
+        author: peer1,
+        recipient: peer2,
+        sessionId: PublicKey.random(),
+        data: {
+          offer: {}
+        }
+      })
+    ).toBeAnObjectWith({ accept: true });
+
+    expect(
+      await peerNetworking2.messageRouter.offer({
+        topic,
+        author: peer2,
+        recipient: peer1,
+        sessionId: PublicKey.random(),
+        data: {
+          offer: {}
+        }
+      })
+    ).toBeAnObjectWith({ accept: true });
+
+    {
+      const message: SignalMessage = {
+        topic,
+        author: peer1,
+        recipient: peer2,
+        sessionId: PublicKey.random(),
+        data: {
+          signal: { json: JSON.stringify({ foo: 'bar' }) }
+        }
+      };
+      await peerNetworking1.messageRouter.signal(message);
+
+      await waitForExpect(() => {
+        expect(peerNetworking2.receivedSignals[0]).toBeAnObjectWith(message);
+      });
+    }
+
+    {
+      const message: SignalMessage = {
+        topic,
+        author: peer2,
+        recipient: peer1,
+        sessionId: PublicKey.random(),
+        data: {
+          signal: { json: JSON.stringify({ foo: 'bar' }) }
+        }
+      };
+      await peerNetworking2.messageRouter.signal(message);
+
+      await waitForExpect(() => {
+        expect(peerNetworking1.receivedSignals[0]).toBeAnObjectWith(message);
+      });
+    }
+  });
+});