@dxos/network-manager 0.1.56-main.e47dfd1 → 0.1.56-main.e79d64a

package/src/swarm/swarm.ts

@@ -2,11 +2,10 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
 import { Event, scheduleTask, sleep, synchronized } from '@dxos/async';
 import { Context } from '@dxos/context';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import { ListeningHandle, Messenger } from '@dxos/messaging';
@@ -35,17 +34,17 @@ const getClassName = (obj: any) => Object.getPrototypeOf(obj).constructor.name;
  * Routes signal events and maintains swarm topology.
  */
 export class Swarm {
-  /**
-   * @internal
-   */
-  readonly _peers = new ComplexMap<PublicKey, Peer>(PublicKey.hash);
-
   private readonly _swarmMessenger: SwarmMessenger;
 
   private _ctx = new Context();
 
   private _listeningHandle?: ListeningHandle = undefined;
 
+  /**
+   * @internal
+   */
+  readonly _peers = new ComplexMap<PublicKey, Peer>(PublicKey.hash);
+
   /**
    * Unique id of the swarm, local to the current peer, generated when swarm is joined.
    */

package/src/testing/test-builder.ts

@@ -24,7 +24,7 @@ import {
   WebRTCTransportProxyFactory,
   WebRTCTransportService,
 } from '../transport';
-import { TestWireProtocol } from './test-wire-protocol';
+import { TestWireProtocol, type TestTeleportExtensionFactory } from './test-wire-protocol';
 
 // Signal server will be started by the setup script.
 const port = process.env.SIGNAL_PORT ?? 4000;
@@ -152,13 +152,13 @@ export class TestPeer {
     return swarm;
   }
 
-  createSwarm(topic: PublicKey): TestSwarmConnection {
+  createSwarm(topic: PublicKey, extensionFactory: TestTeleportExtensionFactory = () => []): TestSwarmConnection {
     // TODO(burdon): Multiple.
     // if (this._swarms.get(topic)) {
     //   throw new Error(`Swarm already exists for topic: ${topic.truncate()}`);
     // }
 
-    const swarm = new TestSwarmConnection(this, topic);
+    const swarm = new TestSwarmConnection(this, topic, extensionFactory);
     this._swarms.set(topic, swarm);
     return swarm;
   }
@@ -176,10 +176,14 @@ export class TestPeer {
 export class TestSwarmConnection {
   protocol: TestWireProtocol;
 
-  constructor(readonly peer: TestPeer, readonly topic: PublicKey) {
+  constructor(
+    readonly peer: TestPeer,
+    readonly topic: PublicKey,
+    readonly extensionFactory: TestTeleportExtensionFactory,
+  ) {
     // TODO(burdon): Configure plugins.
     // TODO(burdon): Prevent reuse?
-    this.protocol = new TestWireProtocol(this.peer.peerId);
+    this.protocol = new TestWireProtocol(this.peer.peerId, this.extensionFactory);
   }
 
   // TODO(burdon): Need to create new plugin instance per swarm?

package/src/testing/test-wire-protocol.ts

@@ -5,11 +5,19 @@
 import { asyncTimeout, Event } from '@dxos/async';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { TestExtension, TestExtensionWithStreams, type TestStreamStats } from '@dxos/teleport';
+import { TestExtension, TestExtensionWithStreams } from '@dxos/teleport';
+import type { TestStreamStats, TeleportExtension } from '@dxos/teleport';
 import { ComplexMap } from '@dxos/util';
 
 import { createTeleportProtocolFactory } from '../wire-protocol';
 
+export type TestTeleportExtension = {
+  name: string;
+  extension: TeleportExtension;
+};
+
+export type TestTeleportExtensionFactory = () => TestTeleportExtension[];
+
 export class TestWireProtocol {
   public readonly connections = new ComplexMap<PublicKey, TestExtension>(PublicKey.hash);
   public readonly streamConnections = new ComplexMap<PublicKey, TestExtensionWithStreams>(PublicKey.hash);
@@ -17,7 +25,14 @@ export class TestWireProtocol {
   public readonly connected = new Event<PublicKey>();
   public readonly disconnected = new Event<PublicKey>();
 
-  constructor(public readonly peerId: PublicKey) {}
+  public readonly otherConnections = new ComplexMap<{ remotePeerId: PublicKey; extension: string }, TeleportExtension>(
+    ({ remotePeerId, extension }) => remotePeerId.toHex() + extension,
+  );
+
+  constructor(
+    public readonly peerId: PublicKey,
+    private readonly _extensionFactory: TestTeleportExtensionFactory = () => [],
+  ) {}
 
   readonly factory = createTeleportProtocolFactory(async (teleport) => {
     log('create', { remotePeerId: teleport.remotePeerId });
@@ -38,6 +53,11 @@ export class TestWireProtocol {
     });
     this.streamConnections.set(teleport.remotePeerId, streamExtension);
     teleport.addExtension('test-stream', streamExtension);
+
+    for (const { name, extension } of this._extensionFactory()) {
+      this.otherConnections.set({ remotePeerId: teleport.remotePeerId, extension: name }, extension);
+      teleport.addExtension(name, extension);
+    }
   });
 
   async waitForConnection(peerId: PublicKey) {

package/src/tests/memory-transport.test.ts

@@ -45,7 +45,7 @@ describe('Memory transport', () => {
       const message = randomBytes(PACKET_SIZE / 2).toString('hex');
 
       for (let i = 0; i < NUM_ROUNDS; i++) {
-        console.log(`Round ${i}/${NUM_ROUNDS}`);
+        // console.log(`Round ${i}/${NUM_ROUNDS}`);
         await Promise.all(
           pairs.map(async ({ swarm1, swarm2 }) => {
             await exchangeMessages(swarm1, swarm2, message);

package/src/topology/fully-connected-topology.ts

@@ -2,8 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 
 import { SwarmController, Topology } from './topology';

package/src/topology/mmst-topology.ts

@@ -2,9 +2,9 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
 import distance from 'xor-distance';
 
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 

package/src/topology/star-topology.ts

@@ -2,8 +2,7 @@
 // Copyright 2020 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
-
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 

package/src/transport/memory-transport.ts

@@ -3,10 +3,10 @@
 //
 
 import { Transform } from 'node:stream';
-import invariant from 'tiny-invariant';
 
 import { Event, Trigger } from '@dxos/async';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log, logInfo } from '@dxos/log';
 import { Signal } from '@dxos/protocols/proto/dxos/mesh/swarm';

package/src/transport/webrtc-transport-proxy.ts

@@ -2,12 +2,13 @@
 // Copyright 2022 DXOS.org
 //
 
-import invariant from 'tiny-invariant';
+import { Writable } from 'node:stream';
 
 import { Event } from '@dxos/async';
 import { Stream } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
 import { ErrorStream } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { ConnectionState, BridgeEvent, BridgeService } from '@dxos/protocols/proto/dxos/mesh/bridge';
@@ -20,6 +21,7 @@ export type WebRTCTransportProxyParams = {
   initiator: boolean;
   stream: NodeJS.ReadWriteStream;
   bridgeService: BridgeService;
+  // TODO(burdon): Rename onSignal.
   sendSignal: (signal: Signal) => Promise<void>;
 };
 
@@ -28,11 +30,10 @@ export class WebRTCTransportProxy implements Transport {
   private readonly _ctx = new Context();
 
   readonly closed = new Event();
-  private _closed = false;
   readonly connected = new Event();
-
   readonly errors = new ErrorStream();
 
+  private _closed = false;
   private _serviceStream!: Stream<BridgeEvent>;
 
   // prettier-ignore
@@ -57,18 +58,19 @@ export class WebRTCTransportProxy implements Transport {
           }
         });
 
-        const dataListener = async (data: Uint8Array) => {
-          try {
-            await this._params.bridgeService.sendData({
+        this._params.stream.pipe(new Writable({
+          write: (chunk, _, callback) => {
+            this._params.bridgeService.sendData({
               proxyId: this._proxyId,
-              payload: data
-            });
-          } catch (err: any) {
-            log.catch(err);
-          }
-        };
-        this._params.stream.on('data', dataListener);
-        this._ctx.onDispose(() => { this._params.stream.off('data', dataListener); });
+              payload: chunk
+            }).then(
+              () => callback(),
+              (err: any) => {
+                log.catch(err);
+              }
+            );
+          },
+        }));
       },
       (error) => log.catch(error)
     );
@@ -116,7 +118,7 @@ export class WebRTCTransportProxy implements Transport {
       return;
     }
 
-    this._serviceStream.close();
+    await this._serviceStream.close();
 
     try {
       await this._params.bridgeService.close({ proxyId: this._proxyId });
@@ -133,7 +135,7 @@ export class WebRTCTransportProxy implements Transport {
    */
   // TODO(burdon): Option on close method.
   forceClose() {
-    this._serviceStream.close();
+    void this._serviceStream.close();
     this.closed.emit();
     this._closed = true;
   }

package/src/transport/webrtc-transport-service.ts

@@ -3,9 +3,9 @@
 //
 
 import { Duplex } from 'node:stream';
-import invariant from 'tiny-invariant';
 
 import { Stream } from '@dxos/codec-protobuf';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import {
@@ -21,20 +21,30 @@ import { ComplexMap } from '@dxos/util';
 
 import { WebRTCTransport } from './webrtc-transport';
 
+type TransportState = {
+  transport: WebRTCTransport;
+  stream: Duplex;
+  writeCallbacks: (() => void)[];
+};
+
 export class WebRTCTransportService implements BridgeService {
-  private readonly transports = new ComplexMap<PublicKey, { transport: WebRTCTransport; stream: Duplex }>(
-    PublicKey.hash,
-  );
+  private readonly transports = new ComplexMap<PublicKey, TransportState>(PublicKey.hash);
 
   // prettier-ignore
   constructor(
     private readonly _webrtcConfig?: any
-  ) {}
+  ) { }
 
   open(request: ConnectionRequest): Stream<BridgeEvent> {
     const rpcStream: Stream<BridgeEvent> = new Stream(({ ready, next, close }) => {
       const duplex: Duplex = new Duplex({
-        read: () => {},
+        read: () => {
+          const callbacks = [...transportState.writeCallbacks];
+          transportState.writeCallbacks.length = 0;
+          for (const cb of callbacks) {
+            cb();
+          }
+        },
         write: function (chunk, _, callback) {
           next({ data: { payload: chunk } });
           callback();
@@ -85,8 +95,15 @@ export class WebRTCTransportService implements BridgeService {
         close();
       });
 
+      const transportState: TransportState = {
+        transport,
+        stream: duplex,
+        writeCallbacks: [],
+      };
+
       ready();
-      this.transports.set(request.proxyId, { transport, stream: duplex });
+
+      this.transports.set(request.proxyId, transportState);
     });
 
     return rpcStream;
@@ -99,7 +116,13 @@ export class WebRTCTransportService implements BridgeService {
 
   async sendData({ proxyId, payload }: DataRequest): Promise<void> {
     invariant(this.transports.has(proxyId));
-    await this.transports.get(proxyId)!.stream.push(payload);
+    const state = this.transports.get(proxyId)!;
+    const bufferHasSpace = state.stream.push(payload);
+    if (!bufferHasSpace) {
+      await new Promise<void>((resolve) => {
+        state.writeCallbacks.push(resolve);
+      });
+    }
   }
 
   async close({ proxyId }: CloseRequest) {

package/src/transport/webrtc-transport.ts

@@ -3,10 +3,10 @@
 //
 
 import SimplePeerConstructor, { Instance as SimplePeer } from 'simple-peer';
-import invariant from 'tiny-invariant';
 
 import { Event } from '@dxos/async';
 import { ErrorStream, raise } from '@dxos/debug';
+import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
 import { trace } from '@dxos/protocols';
@@ -17,31 +17,45 @@ import { wrtc } from './webrtc';
 
 export type WebRTCTransportParams = {
   initiator: boolean;
-  stream: NodeJS.ReadWriteStream;
+  // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/RTCPeerConnection
   webrtcConfig?: any;
+  stream: NodeJS.ReadWriteStream;
+  // TODO(burdon): Rename onSignal.
   sendSignal: (signal: Signal) => Promise<void>;
 };
 
+// TODO(burdon): Refs:
+//  https://webrtc.github.io/samples
+//  Simple peer discord.
+
 /**
- * Implements Transport for WebRTC. Uses simple-peer under the hood.
+ * Implements WebRTC transport using simple-peer (used by WebTorrent).
  */
 export class WebRTCTransport implements Transport {
+  private readonly _instanceId = PublicKey.random().toHex();
   private readonly _peer: SimplePeer;
-  private _closed = false;
 
+  private _closed = false;
   readonly closed = new Event();
   readonly connected = new Event();
   readonly errors = new ErrorStream();
 
-  private readonly _instanceId = PublicKey.random().toHex();
-
   constructor(private readonly params: WebRTCTransportParams) {
     log.trace('dxos.mesh.webrtc-transport.constructor', trace.begin({ id: this._instanceId }));
     log('created connection', params);
+
+    // https://www.npmjs.com/package/simple-peer#api
+    // NOTE: Set localStorage.debug = 'simple-peer*' to enable debug logging. (Or show verbose in Chrome).
+    // View throughput via: chrome://webrtc-internals (data-channel)
     this._peer = new SimplePeerConstructor({
       initiator: this.params.initiator,
-      wrtc: SimplePeerConstructor.WEBRTC_SUPPORT ? undefined : wrtc ?? raise(new Error('wrtc not available')),
       config: this.params.webrtcConfig,
+      // https://developer.mozilla.org/en-US/docs/Web/API/RTCPeerConnection/createDataChannel
+      channelConfig: {
+        protocol: 'dxos.mesh',
+      },
+      // Custom WebRTC implementation (for Node); otherwise get-browser-rtc.
+      wrtc: SimplePeerConstructor.WEBRTC_SUPPORT ? undefined : wrtc ?? raise(new Error('wrtc not available')),
     });
 
     this._peer.on('signal', async (data) => {