@dxos/network-manager 2.33.9-dev.9246a07b → 2.33.9-dev.9bbef4e2

package/src/signal/signal-client.ts

@@ -2,26 +2,55 @@
 // Copyright 2020 DXOS.org
 //
 
+import assert from 'assert';
 import debug from 'debug';
 
-import { Event } from '@dxos/async';
-import { failUndefined } from '@dxos/debug';
+import { Event, synchronized } from '@dxos/async';
+import { Any, Stream } from '@dxos/codec-protobuf';
 import { PublicKey } from '@dxos/protocols';
+import { ComplexMap, SubscriptionGroup } from '@dxos/util';
 
 import { schema } from '../proto/gen';
-import { Answer, Message } from '../proto/gen/dxos/mesh/signal';
+import { Message, SwarmEvent } from '../proto/gen/dxos/mesh/signal';
+import { SignalMessage } from '../proto/gen/dxos/mesh/signalMessage';
 import { SignalApi } from './signal-api';
-import { WebsocketRpc } from './websocket-rpc';
+import { SignalRPCClient } from './signal-rpc-client';
 
 const log = debug('dxos:network-manager:signal-client');
 
 const DEFAULT_RECONNECT_TIMEOUT = 1000;
 
+enum State {
+  /** Connection is being established. */
+  CONNECTING = 'CONNECTING',
+
+  /** Connection is being re-established. */
+  RE_CONNECTING = 'RE_CONNECTING',
+
+  /** Connected. */
+  CONNECTED = 'CONNECTED',
+
+  /** Server terminated the connection. Socket will be reconnected. */
+  DISCONNECTED = 'DISCONNECTED',
+
+  /** Socket was closed. */
+  CLOSED = 'CLOSED'
+}
+
+export type Status = {
+  host: string
+  state: State
+  error?: string
+  reconnectIn: number
+  connectionStarted: number
+  lastStateChange: number
+}
+
 /**
  * Establishes a websocket connection to signal server and provides RPC methods.
  */
 export class SignalClient {
-  private _state = SignalApi.State.CONNECTING;
+  private _state = State.CONNECTING;
 
   private _lastError?: Error;
 
@@ -42,28 +71,29 @@ export class SignalClient {
 
   private _reconnectIntervalId?: NodeJS.Timeout;
 
-  private _client!: WebsocketRpc;
+  private _client!: SignalRPCClient;
 
-  private _clientCleanup: (() => void)[] = [];
+  private _cleanupSubscriptions = new SubscriptionGroup();
+  readonly statusChanged = new Event<Status>();
 
-  readonly statusChanged = new Event<SignalApi.Status>();
   readonly commandTrace = new Event<SignalApi.CommandTrace>();
+  readonly swarmEvent = new Event<[topic: PublicKey, swarmEvent: SwarmEvent]>();
 
+  private readonly _swarmStreams = new ComplexMap<PublicKey, Stream<SwarmEvent>>(key => key.toHex());
+  private readonly _messageStreams = new ComplexMap<PublicKey, Stream<Message>>(key => key.toHex());
   /**
    * @param _host Signal server websocket URL.
-   * @param _onOffer See `SignalApi.offer`.
    * @param _onSignal See `SignalApi.signal`.
    */
   constructor (
     private readonly _host: string,
-    private readonly _onOffer: (message: Message) => Promise<Answer>,
-    private readonly _onSignal: (message: Message) => Promise<void>
+    private readonly _onSignal: (message: SignalMessage) => Promise<void>
   ) {
-    this._setState(SignalApi.State.CONNECTING);
+    this._setState(State.CONNECTING);
     this._createClient();
   }
 
-  private _setState (newState: SignalApi.State) {
+  private _setState (newState: State) {
     this._state = newState;
     this._lastStateChange = Date.now();
     log(`Signal state changed ${JSON.stringify(this.getStatus())}`);
@@ -73,111 +103,88 @@ export class SignalClient {
   private _createClient () {
     this._connectionStarted = Date.now();
     try {
-      this._client = new WebsocketRpc(this._host);
+      this._client = new SignalRPCClient(this._host);
     } catch (error: any) {
-      if (this._state === SignalApi.State.RE_CONNECTING) {
+      if (this._state === State.RE_CONNECTING) {
         this._reconnectAfter *= 2;
       }
 
       this._lastError = error;
-      this._setState(SignalApi.State.DISCONNECTED);
+      this._setState(State.DISCONNECTED);
       this._reconnect();
     }
 
-    this._client.addHandler('offer', (message: any) => this._onOffer({
-      id: PublicKey.from(message.id),
-      remoteId: PublicKey.from(message.remoteId),
-      topic: PublicKey.from(message.topic),
-      sessionId: PublicKey.from(message.sessionId),
-      data: message.data
-    }));
-
-    this._client.subscribe('signal', (msg: SignalMessage) => {
-      return this._onSignal({
-        id: PublicKey.from(msg.id!),
-        remoteId: PublicKey.from(msg.remoteId!),
-        topic: PublicKey.from(msg.topic!),
-        sessionId: PublicKey.from(msg.sessionId!),
-        data: schema.getCodecForType('dxos.mesh.signal.MessageData').decode(msg.data ?? failUndefined()),
-        // Field that MessageRouter adds, so on lower level it not always defined.
-        messageId: msg.messageId ? PublicKey.from(msg.messageId) : undefined
-      });
-    });
-
-    this._clientCleanup.push(this._client.connected.on(() => {
-      log('Socket connected');
+    this._cleanupSubscriptions.push(this._client.connected.on(() => {
       this._lastError = undefined;
       this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
-      this._setState(SignalApi.State.CONNECTED);
+      this._setState(State.CONNECTED);
     }));
 
-    this._clientCleanup.push(this._client.error.on(error => {
+    this._cleanupSubscriptions.push(this._client.error.on(error => {
       log(`Socket error: ${error.message}`);
-      if (this._state === SignalApi.State.CLOSED) {
+      if (this._state === State.CLOSED) {
         return;
       }
 
-      if (this._state === SignalApi.State.RE_CONNECTING) {
+      if (this._state === State.RE_CONNECTING) {
         this._reconnectAfter *= 2;
       }
 
       this._lastError = error;
-      this._setState(SignalApi.State.DISCONNECTED);
+      this._setState(State.DISCONNECTED);
 
       this._reconnect();
     }));
 
-    this._clientCleanup.push(this._client.disconnected.on(() => {
+    this._cleanupSubscriptions.push(this._client.disconnected.on(() => {
       log('Socket disconnected');
       // This is also called in case of error, but we already have disconnected the socket on error, so no need to do anything here.
-      if (this._state !== SignalApi.State.CONNECTING && this._state !== SignalApi.State.RE_CONNECTING) {
+      if (this._state !== State.CONNECTING && this._state !== State.RE_CONNECTING) {
         return;
       }
 
-      if (this._state === SignalApi.State.RE_CONNECTING) {
+      if (this._state === State.RE_CONNECTING) {
         this._reconnectAfter *= 2;
       }
 
-      this._setState(SignalApi.State.DISCONNECTED);
+      this._setState(State.DISCONNECTED);
       this._reconnect();
     }));
-
-    this._clientCleanup.push(this._client.commandTrace.on(trace => this.commandTrace.emit(trace)));
   }
 
   private _reconnect () {
+    log(`Reconnecting in ${this._reconnectAfter}ms`);
     if (this._reconnectIntervalId !== undefined) {
       console.error('Signal api already reconnecting.');
       return;
     }
-    if (this._state === SignalApi.State.CLOSED) {
+    if (this._state === State.CLOSED) {
       return;
     }
 
     this._reconnectIntervalId = setTimeout(() => {
       this._reconnectIntervalId = undefined;
 
-      this._clientCleanup.forEach(cb => cb());
-      this._clientCleanup = [];
+      this._cleanupSubscriptions.unsubscribe();
 
       // Close client if it wasn't already closed.
       this._client.close().catch(() => {});
 
-      this._setState(SignalApi.State.RE_CONNECTING);
+      this._setState(State.RE_CONNECTING);
       this._createClient();
     }, this._reconnectAfter);
   }
 
   async close () {
-    this._clientCleanup.forEach(cb => cb());
-    this._clientCleanup = [];
+    this._cleanupSubscriptions.unsubscribe();
 
     if (this._reconnectIntervalId !== undefined) {
       clearTimeout(this._reconnectIntervalId);
     }
 
     await this._client.close();
-    this._setState(SignalApi.State.CLOSED);
+    this._setState(State.CLOSED);
+    log('Closed.');
   }
 
   getStatus (): SignalApi.Status {
@@ -191,73 +198,72 @@ export class SignalClient {
     };
   }
 
-  async join (topic: PublicKey, peerId: PublicKey): Promise<PublicKey[]> {
-    const peers: Buffer[] = await this._client.call('join', {
-      id: peerId.asBuffer(),
-      topic: topic.asBuffer()
-    });
-    return peers.map(id => PublicKey.from(id));
+  async join (topic: PublicKey, peerId: PublicKey): Promise<void> {
+    log(`Join: topic=${topic} peerId=${peerId}`);
+    await this._subscribeMessages(peerId);
+    await this._subscribeSwarmEvents(topic, peerId);
   }
 
   async leave (topic: PublicKey, peerId: PublicKey): Promise<void> {
-    await this._client.call('leave', {
-      id: peerId.asBuffer(),
-      topic: topic.asBuffer()
-    });
+    log(`Leave: topic=${topic} peerId=${peerId}`);
+
+    this._swarmStreams.get(topic)?.close();
+    this._swarmStreams.delete(topic);
+
+    this._messageStreams.get(topic)?.close();
+    this._messageStreams.delete(topic);
   }
 
-  async lookup (topic: PublicKey): Promise<PublicKey[]> {
-    const peers: Buffer[] = await this._client.call('lookup', {
-      topic: topic.asBuffer()
-    });
-    return peers.map(id => PublicKey.from(id));
+  async signal (message: SignalMessage): Promise<void> {
+    const payload: Any = {
+      type_url: 'dxos.mesh.signalMessage.SignalMessage',
+      value: schema.getCodecForType('dxos.mesh.signalMessage.SignalMessage').encode(message)
+    };
+    return this._client.sendMessage(message.id, message.remoteId, payload);
   }
 
-  /**
-   * Routes an offer to the other peer's _onOffer callback.
-   * @returns Other peer's _onOffer callback return value.
-   * @deprecated
-   */
-  async offer (msg: Message): Promise<Answer> {
-    return this._client.call('offer', {
-      id: msg.id?.asBuffer(),
-      remoteId: msg.remoteId?.asBuffer(),
-      topic: msg.topic?.asBuffer(),
-      sessionId: msg.sessionId?.asBuffer(),
-      data: msg.data
+  @synchronized
+  private async _subscribeSwarmEvents (topic: PublicKey, peerId: PublicKey): Promise<void> {
+    assert(!this._swarmStreams.has(topic));
+    const swarmStream = await this._client.join(topic, peerId);
+    // Subscribing to swarm events.
+    // TODO(mykola): What happens when the swarm stream is closed? Maybe send leave event for each peer?
+    swarmStream.subscribe((swarmEvent: SwarmEvent) => {
+      this.swarmEvent.emit([topic, swarmEvent]);
+    });
+
+    // Saving swarm stream.
+    this._swarmStreams.set(topic, swarmStream);
+
+    this._cleanupSubscriptions.push(() => {
+      swarmStream.close();
+      this._swarmStreams.delete(topic);
     });
   }
 
-  /**
-   * Routes an offer to the other peer's _onSignal callback.
-   */
-  async signal (message: Message): Promise<void> {
-    const signalMessage: SignalMessage = {
-      messageId: message.messageId?.asBuffer(),
-      id: message.id?.asBuffer(),
-      remoteId: message.remoteId?.asBuffer(),
-      topic: message.topic?.asBuffer(),
-      sessionId: message.sessionId?.asBuffer(),
-      data: Buffer.from(schema.getCodecForType('dxos.mesh.signal.MessageData').encode(message.data ?? failUndefined()))
-    };
-    return this._client.emit('signal', signalMessage);
+  private async _subscribeMessages (peerId: PublicKey) {
+    // Subscribing to messages.
+    const messageStream = await this._client.receiveMessages(peerId);
+    messageStream.subscribe(async (message: Message) => {
+      if (message.payload.type_url === 'dxos.mesh.signalMessage.SignalMessage') {
+        const signalMessage = schema.getCodecForType('dxos.mesh.signalMessage.SignalMessage').decode(message.payload.value);
+        log('Message received: ' + JSON.stringify(signalMessage));
+        assert(signalMessage.remoteId.equals(peerId));
+        await this._onSignal(signalMessage);
+      } else {
+        log('Unknown message type: ' + message.payload.type_url);
+      }
+    });
+
+    // Saving message stream.
+    if (!this._messageStreams.has(peerId)) {
+      this._messageStreams.set(peerId, messageStream);
+    }
+
+    this._cleanupSubscriptions.push(() => {
+      messageStream.close();
+      this._messageStreams.delete(peerId);
+    });
   }
-}
 
-/**
- * Messages as processed by the signal server.
- */
-interface SignalMessage{
-  /**
-   * Sender's public key.
-   */
-  id?: Buffer;
-  /**
-   * Receiver`s public key.
-   */
-  remoteId?: Buffer;
-  topic?: Buffer;
-  sessionId?: Buffer;
-  data?: Buffer;
-  messageId?: Buffer;
 }

package/src/signal/signal-manager-impl.ts

@@ -0,0 +1,166 @@
+//
+// Copyright 2020 DXOS.org
+//
+
+import debug from 'debug';
+import assert from 'node:assert';
+
+import { Event, synchronized } from '@dxos/async';
+import { PublicKey } from '@dxos/protocols';
+import { ComplexMap } from '@dxos/util';
+
+import { SwarmEvent } from '../proto/gen/dxos/mesh/signal';
+import { SignalMessage } from '../proto/gen/dxos/mesh/signalMessage';
+import { SignalApi } from './signal-api';
+import { SignalClient } from './signal-client';
+import { SignalManager } from './signal-manager';
+
+const log = debug('dxos:network-manager:signal-manager-impl');
+
+export class SignalManagerImpl implements SignalManager {
+  private readonly _servers = new Map<string, SignalClient>();
+
+  /** Topics joined: topic => peerId */
+  private readonly _topicsJoined = new ComplexMap<PublicKey, PublicKey>(topic => topic.toHex());
+  /** host => topic => peerId */
+  private readonly _topicsJoinedPerSignal = new Map<string, ComplexMap<PublicKey, PublicKey>>();
+
+  private _reconciling?: boolean = false;
+  private _reconcileTimeoutId?: NodeJS.Timeout;
+  private _destroyed = false;
+
+  readonly statusChanged = new Event<SignalApi.Status[]>();
+  readonly commandTrace = new Event<SignalApi.CommandTrace>();
+  readonly swarmEvent = new Event<[topic: PublicKey, swarmEvent: SwarmEvent]>();
+  readonly onMessage = new Event<SignalMessage>();
+
+  constructor (
+    private readonly _hosts: string[]
+  ) {
+    log(`Created WebsocketSignalManager with signal servers: ${_hosts}`);
+    assert(_hosts.length === 1, 'Only a single signaling server connection is supported');
+    for (const host of this._hosts) {
+      const server = new SignalClient(
+        host,
+        async msg => this.onMessage.emit(msg)
+      );
+      // TODO(mykola): Add subscription group
+      server.swarmEvent.on(data => this.swarmEvent.emit(data));
+      server.statusChanged.on(() => this.statusChanged.emit(this.getStatus()));
+
+      this._servers.set(host, server);
+      server.commandTrace.on(trace => this.commandTrace.emit(trace));
+      this._topicsJoinedPerSignal.set(host, new ComplexMap(x => x.toHex()));
+    }
+  }
+
+  getStatus (): SignalApi.Status[] {
+    return Array.from(this._servers.values()).map(server => server.getStatus());
+  }
+
+  join (topic: PublicKey, peerId: PublicKey) {
+    assert(!this._topicsJoined.has(topic), `Topic ${topic} is already joined`);
+    log(`Join ${topic} ${peerId}`);
+    this._topicsJoined.set(topic, peerId);
+    this._scheduleReconcile();
+  }
+
+  leave (topic: PublicKey, peerId: PublicKey) {
+    assert(!!this._topicsJoined.has(topic), `Topic ${topic} was not joined`);
+    log(`Leave ${topic} ${peerId}`);
+    this._topicsJoined.delete(topic);
+    this._scheduleReconcile();
+  }
+
+  private _scheduleReconcile () {
+    if (this._destroyed) {
+      return;
+    }
+
+    if (!this._reconciling) {
+      this._reconciling = true;
+      this._reconcileJoinedTopics().then(
+        () => {
+          this._reconciling = false;
+        },
+        err => {
+          this._reconciling = false;
+          log(`Error while reconciling: ${err}`);
+          this._reconcileLater();
+        }
+      );
+    } else {
+      this._reconcileLater();
+    }
+  }
+
+  private _reconcileLater () {
+    if (this._destroyed) {
+      return;
+    }
+
+    if (!this._reconcileTimeoutId) {
+      this._reconcileTimeoutId = setTimeout(async () => this._scheduleReconcile(), 3000);
+    }
+  }
+
+  @synchronized
+  private async _reconcileJoinedTopics () {
+    // TODO(mykola): Handle reconnects to SS. Maybe move map of joined topics to signal-client.
+    log('Reconciling..');
+    for (const [host, server] of this._servers) {
+      const actualJoinedTopics = this._topicsJoinedPerSignal.get(host)!;
+
+      // Leave swarms
+      for (const [topic, actualPeerId] of actualJoinedTopics) {
+        try {
+          const desiredPeerId = this._topicsJoined.get(topic);
+          if (!desiredPeerId || !desiredPeerId.equals(actualPeerId)) {
+            await server.leave(topic, actualPeerId);
+            actualJoinedTopics.delete(topic);
+          }
+        } catch (err) {
+          log(`Error leaving swarm: ${err}`);
+          this._scheduleReconcile();
+        }
+      }
+
+      // Join swarms
+      for (const [topic, desiredPeerId] of this._topicsJoined) {
+        try {
+          const actualPeerId = actualJoinedTopics.get(topic);
+          if (!actualPeerId) {
+            await server.join(topic, desiredPeerId);
+            actualJoinedTopics.set(topic, desiredPeerId);
+          } else {
+            if (!actualPeerId.equals(desiredPeerId)) {
+              throw new Error(`Joined with peerId different from desired: ${JSON.stringify({ actualPeerId, desiredPeerId })}`);
+            }
+          }
+        } catch (err) {
+          log(`Error joining swarm: ${err}`);
+          this._scheduleReconcile();
+        }
+      }
+    }
+    log('Done reconciling..');
+    this._reconciling = false;
+  }
+
+  async message (msg: SignalMessage) {
+    log(`Signal ${msg.remoteId}`);
+    for (const server of this._servers.values()) {
+      server.signal(msg).catch(err => {
+        log(`Error signaling: ${err}`);
+      });
+    }
+  }
+
+  async destroy () {
+    this._destroyed = true;
+    if (this._reconcileTimeoutId) {
+      clearTimeout(this._reconcileTimeoutId);
+    }
+    await Promise.all(Array.from(this._servers.values()).map(server => server.close()));
+  }
+}

package/src/signal/signal-manager.ts

@@ -5,18 +5,14 @@
 import { Event } from '@dxos/async';
 import { PublicKey } from '@dxos/protocols';
 
-import { Answer, Message } from '../proto/gen/dxos/mesh/signal';
+import { SwarmEvent } from '../proto/gen/dxos/mesh/signal';
+import { Answer, SignalMessage } from '../proto/gen/dxos/mesh/signalMessage';
 import { SignalApi } from './signal-api';
 
 /**
  * Signal peer discovery interface.
  */
 export interface SignalConnection {
-  /**
-   * Find peers (triggers async event).
-   */
-  lookup (topic: PublicKey): void
-
   /**
    * Join topic on signal network, to be discoverable by other peers.
    */
@@ -35,20 +31,24 @@ export interface SignalMessaging {
   /**
    * Offer/answer RPC.
    */
-  offer (msg: Message): Promise<Answer>
+  offer (msg: SignalMessage): Promise<Answer>
 
   /**
-   * Send message to peer.
+   * Reliably send a signal to a peer.
    */
-  signal (msg: Message): Promise<void>
+  signal (msg: SignalMessage): Promise<void>
 }
 
-export interface SignalManager extends SignalConnection, SignalMessaging {
+export interface SignalManager extends SignalConnection {
   statusChanged: Event<SignalApi.Status[]>
   commandTrace: Event<SignalApi.CommandTrace>
-  peerCandidatesChanged: Event<[topic: PublicKey, candidates: PublicKey[]]>
-  onSignal: Event<Message>
+  swarmEvent: Event<[topic: PublicKey, swarmEvent: SwarmEvent]>
+  onMessage: Event<SignalMessage>
 
   getStatus (): SignalApi.Status[]
   destroy(): Promise<void>
+  /**
+   * Send message to peer.
+   */
+  message (msg: SignalMessage): Promise<void>
 }

package/src/signal/signal-rpc-client.test.ts

@@ -0,0 +1,86 @@
+//
+// Copyright 2022 DXOS.org
+//
+import { expect } from 'earljs';
+
+import { Any } from '@dxos/codec-protobuf';
+import { PublicKey } from '@dxos/protocols';
+import { createTestBroker, TestBroker } from '@dxos/signal';
+
+import { Message, SwarmEvent } from '../proto/gen/dxos/mesh/signal';
+import { SignalRPCClient } from './signal-rpc-client';
+
+describe('SignalRPCClient', () => {
+  let broker: TestBroker;
+
+  before(async () => {
+    broker = await createTestBroker();
+  });
+
+  after(() => {
+    broker.stop();
+  });
+
+  const setup = async () => {
+    const client = new SignalRPCClient(broker.url());
+    return client;
+  };
+
+  it('signal between 2 peers', async () => {
+    const client1 = await setup();
+    const client2 = await setup();
+
+    const peerId1 = PublicKey.random();
+    const peerId2 = PublicKey.random();
+
+    const stream1 = await client1.receiveMessages(peerId1);
+    const message: Any = {
+      'type_url': 'test',
+      value: Uint8Array.from([1, 2, 3])
+    };
+
+    await client2.sendMessage(peerId2, peerId1, message);
+
+    const received: Message = await new Promise(resolve => {
+      stream1.subscribe(message => {
+        resolve(message);
+      }, (error) => {
+        if (error) {
+          console.log(error);
+          throw error;
+        }
+      });
+    });
+    expect(received.author).toEqual(peerId2.asUint8Array());
+    stream1.close();
+  }).timeout(10000);
+
+  it('join', async () => {
+    const client1 = await setup();
+    const client2 = await setup();
+
+    const peerId1 = PublicKey.random();
+    const peerId2 = PublicKey.random();
+    const topic = PublicKey.random();
+
+    const stream1 = await client1.join(topic, peerId1);
+    const promise = new Promise<SwarmEvent>(resolve => {
+      stream1.subscribe((event: SwarmEvent) => {
+        // eslint-disable-next-line @typescript-eslint/no-non-null-asserted-optional-chain
+        if (peerId2.equals(event.peerAvailable?.peer!)) {
+          resolve(event);
+        }
+      }, (error) => {
+        if (error) {
+          console.log(error);
+          throw error;
+        }
+      });
+    });
+    const stream2 = await client2.join(topic, peerId2);
+
+    expect((await promise).peerAvailable?.peer).toEqual(peerId2.asBuffer());
+    stream1.close();
+    stream2.close();
+  }).timeout(10000);
+});