@dxos/network-manager 2.33.9-dev.7d11f506 → 2.33.9-dev.9bbef4e2

package/src/signal/message-router.test.ts

@@ -10,9 +10,8 @@ import { Awaited } from '@dxos/async';
 import { PublicKey } from '@dxos/protocols';
 import { createTestBroker } from '@dxos/signal';
 import { afterTest } from '@dxos/testutils';
-import { randomInt } from '@dxos/util';
 
-import { Answer, Message } from '../proto/gen/dxos/mesh/signal';
+import { Answer, SignalMessage } from '../proto/gen/dxos/mesh/signalMessage';
 import { MessageRouter } from './message-router';
 import { SignalClient } from './signal-client';
 
@@ -22,11 +21,9 @@ describe('MessageRouter', () => {
   let peer2: PublicKey;
 
   let broker1: Awaited<ReturnType<typeof createTestBroker>>;
-  const signalApiPort1 = randomInt(10000, 50000);
-  const signalApiUrl1 = 'http://0.0.0.0:' + signalApiPort1;
 
   before(async () => {
-    broker1 = await createTestBroker(signalApiPort1);
+    broker1 = await createTestBroker();
   });
 
   beforeEach(() => {
@@ -35,30 +32,33 @@ describe('MessageRouter', () => {
     peer2 = PublicKey.random();
   });
 
-  after(async function () {
-    this.timeout(0);
-    await broker1.stop();
+  after(() => {
+    broker1.stop();
   });
 
-  const createSignalClientAndMessageRouter = async (
-    signalApiUrl: string,
-    onSignal: (msg: Message) => Promise<void> = (async () => {}) as any,
-    onOffer: (msg: Message) => Promise<Answer> = async () => ({ accept: true })
-  ) => {
+  const createSignalClientAndMessageRouter = async ({
+    signalApiUrl,
+    onSignal = (async () => { }) as any,
+    onOffer = async () => ({ accept: true })
+  }: {
+    signalApiUrl: string;
+    onSignal?: (msg: SignalMessage) => Promise<void>;
+    onOffer?: (msg: SignalMessage) => Promise<Answer>;
+  }) => {
 
     // eslint-disable-next-line prefer-const
     let api: SignalClient;
     const router: MessageRouter = new MessageRouter({
       // todo(mykola): added catch to avoid not finished request.
-      sendMessage: (msg: Message) => api.signal(msg).catch((_) => { }),
+      sendMessage: (msg: SignalMessage) => api.signal(msg).catch((_) => { }),
       onSignal: onSignal,
       onOffer: onOffer
     });
+    afterTest(() => router.destroy());
 
     api = new SignalClient(
       signalApiUrl,
-      (async () => {}) as any,
-      async (msg: Message) => router.receiveMessage(msg)
+      async (msg: SignalMessage) => router.receiveMessage(msg)
     );
 
     afterTest(() => api.close());
@@ -69,19 +69,19 @@ describe('MessageRouter', () => {
   };
 
   test('signaling between 2 clients', async () => {
-    const signalMock1 = mockFn<(msg: Message) => Promise<void>>().resolvesTo();
-    const { api: api1 } = await createSignalClientAndMessageRouter(signalApiUrl1, signalMock1);
-    const { api: api2, router: router2 } = await createSignalClientAndMessageRouter(signalApiUrl1);
+    const signalMock1 = mockFn<(msg: SignalMessage) => Promise<void>>().resolvesTo();
+    const { api: api1 } = await createSignalClientAndMessageRouter({ signalApiUrl: broker1.url(), onSignal: signalMock1 });
+    const { api: api2, router: router2 } = await createSignalClientAndMessageRouter({ signalApiUrl: broker1.url() });
 
     await api1.join(topic, peer1);
     await api2.join(topic, peer2);
 
-    const msg: Message = {
+    const msg: SignalMessage = {
       id: peer2,
       remoteId: peer1,
       sessionId: PublicKey.random(),
       topic,
-      data: { signal: { json: '{"asd": "asd"}' } }
+      data: { signal: { json: JSON.stringify({ 'asd': 'asd' }) } }
     };
     await router2.signal(msg);
 
@@ -92,15 +92,19 @@ describe('MessageRouter', () => {
 
   test('offer/answer', async () => {
     const { api: api1, router: router1 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      (async () => {}) as any,
-      async () => ({ accept: true })
-    );
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: (async () => { }) as any,
+        onOffer:
+          async () => ({ accept: true })
+      });
     const { api: api2 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      (async () => {}) as any,
-      async () => ({ accept: true })
-    );
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: (async () => { }) as any,
+        onOffer:
+          async () => ({ accept: true })
+      });
 
     await api1.join(topic, peer1);
     await api2.join(topic, peer2);
@@ -112,28 +116,34 @@ describe('MessageRouter', () => {
       topic,
       data: { offer: { } }
     });
-    expect(answer).toEqual({ accept: true });
+    expect(answer.accept).toEqual(true);
   }).timeout(5_000);
 
   test('signaling between 3 clients', async () => {
-    const signalMock1 = mockFn<(msg: Message) => Promise<void>>().resolvesTo();
+    const signalMock1 = mockFn<(msg: SignalMessage) => Promise<void>>().resolvesTo();
     const { api: api1, router: router1 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      signalMock1,
-      async () => ({ accept: true })
-    );
-    const signalMock2 = mockFn<(msg: Message) => Promise<void>>().resolvesTo();
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: signalMock1,
+        onOffer:
+          async () => ({ accept: true })
+      });
+    const signalMock2 = mockFn<(msg: SignalMessage) => Promise<void>>().resolvesTo();
     const { api: api2, router: router2 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      signalMock2,
-      async () => ({ accept: true })
-    );
-    const signalMock3 = mockFn<(msg: Message) => Promise<void>>().resolvesTo();
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: signalMock2,
+        onOffer:
+          async () => ({ accept: true })
+      });
+    const signalMock3 = mockFn<(msg: SignalMessage) => Promise<void>>().resolvesTo();
     const { api: api3, router: router3 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      signalMock3,
-      async () => ({ accept: true })
-    );
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: signalMock3,
+        onOffer:
+          async () => ({ accept: true })
+      });
 
     await api1.join(topic, peer1);
     await api2.join(topic, peer2);
@@ -141,7 +151,7 @@ describe('MessageRouter', () => {
     await api3.join(topic, peer3);
 
     // sending signal from peer1 to peer3.
-    const msg1to3: Message = {
+    const msg1to3: SignalMessage = {
       id: peer1,
       remoteId: peer3,
       sessionId: PublicKey.random(),
@@ -154,7 +164,7 @@ describe('MessageRouter', () => {
     }, 4_000);
 
     // sending signal from peer2 to peer3.
-    const msg2to3: Message = {
+    const msg2to3: SignalMessage = {
       id: peer2,
       remoteId: peer3,
       sessionId: PublicKey.random(),
@@ -167,7 +177,7 @@ describe('MessageRouter', () => {
     }, 4_000);
 
     // sending signal from peer3 to peer1.
-    const msg3to1: Message = {
+    const msg3to1: SignalMessage = {
       id: peer3,
       remoteId: peer1,
       sessionId: PublicKey.random(),
@@ -182,15 +192,19 @@ describe('MessageRouter', () => {
 
   test('two offers', async () => {
     const { api: api1, router: router1 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      (async () => {}) as any,
-      async () => ({ accept: true })
-    );
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: (async () => { }) as any,
+        onOffer:
+          async () => ({ accept: true })
+      });
     const { api: api2, router: router2 } = await createSignalClientAndMessageRouter(
-      signalApiUrl1,
-      (async () => {}) as any,
-      async () => ({ accept: true })
-    );
+      {
+        signalApiUrl: broker1.url(),
+        onSignal: (async () => { }) as any,
+        onOffer:
+          async () => ({ accept: true })
+      });
 
     await api1.join(topic, peer1);
     await api2.join(topic, peer2);
@@ -203,7 +217,7 @@ describe('MessageRouter', () => {
       topic,
       data: { offer: {} }
     });
-    expect(answer1).toEqual({ accept: true });
+    expect(answer1.accept).toEqual(true);
 
     // sending offer from peer2 to peer1.
     const answer2 = await router2.offer({
@@ -213,6 +227,103 @@ describe('MessageRouter', () => {
       topic,
       data: { offer: {} }
     });
-    expect(answer2).toEqual({ accept: true });
+    expect(answer2.accept).toEqual(true);
   }).timeout(5_000);
+
+  describe('Reliability', () => {
+    const setup = ({
+      onSignal1 = async () => { },
+      onSignal2 = async () => { },
+      // Imitates signal network disruptions (e. g. message doubling, ).
+      messageDisruption = msg => [msg]
+    }: {
+      onSignal1?: (msg: SignalMessage) => Promise<void>;
+      onSignal2?: (msg: SignalMessage) => Promise<void>;
+      messageDisruption?: (msg: SignalMessage) => SignalMessage[];
+    }): {mr1: MessageRouter; mr2: MessageRouter} => {
+
+      const mr1: MessageRouter = new MessageRouter({
+        sendMessage: async msg => messageDisruption(msg).forEach(msg => mr2.receiveMessage(msg)),
+        onOffer: async () => ({ accept: true }),
+        onSignal: onSignal1
+      });
+      afterTest(() => mr1.destroy());
+
+      const mr2: MessageRouter = new MessageRouter({
+        sendMessage: async msg => messageDisruption(msg).forEach(msg => mr1.receiveMessage(msg)),
+        onOffer: async () => ({ accept: true }),
+        onSignal: onSignal2
+      });
+      afterTest(() => mr1.destroy());
+
+      return { mr1, mr2 };
+    };
+
+    test('signaling with non reliable connection', async () => {
+      // Simulate unreliable connection.
+      // Only each 3rd message is sent.
+      let i = 0;
+      const unreliableConnection = (msg: SignalMessage): SignalMessage[] => {
+        i++;
+        if (i % 3 !== 0) {
+          return [msg];
+        }
+        return [];
+      };
+
+      const received: SignalMessage[] = [];
+      const signalMock1 = async (msg: SignalMessage) => {
+        received.push(msg);
+      };
+
+      const { mr2 } = await setup({
+        onSignal1: signalMock1,
+        messageDisruption: unreliableConnection
+      });
+
+      // Sending 3 messages.
+      // Setup sends messages directly to between. So we don`t need to specify any ids.
+      Array(3).fill(0).forEach(async () => {
+        await mr2.signal({
+          id: PublicKey.random(),
+          remoteId: PublicKey.random(),
+          sessionId: PublicKey.random(),
+          topic: PublicKey.random(),
+          data: { signal: { json: JSON.stringify({ 'asd': 'asd' }) } }
+        });
+      });
+      // expect to receive 3 messages.
+      await waitForExpect(() => {
+        expect(received.length).toEqual(3);
+      }, 4_000);
+    }).timeout(5_000);
+
+    test('ignoring doubled messages', async () => {
+      // Message got doubled going through signal network.
+      const doublingMessage = (msg: SignalMessage) => [msg, msg];
+
+      const received: SignalMessage[] = [];
+      const signalMock1 = async (msg: SignalMessage) => {
+        received.push(msg);
+      };
+
+      const { mr2 } = setup({
+        onSignal1: signalMock1,
+        messageDisruption: doublingMessage
+      });
+
+      // sending message.
+      await mr2.signal({
+        id: PublicKey.random(),
+        remoteId: PublicKey.random(),
+        sessionId: PublicKey.random(),
+        topic: PublicKey.random(),
+        data: { signal: { json: 'asd' } }
+      });
+      // expect to receive 1 message.
+      await waitForExpect(() => {
+        expect(received.length).toEqual(1);
+      }, 4_000);
+    }).timeout(5_000);
+  });
 });

package/src/signal/message-router.ts

@@ -3,11 +3,12 @@
 //
 
 import assert from 'assert';
+import debug from 'debug';
 
 import { PublicKey } from '@dxos/protocols';
-import { ComplexMap } from '@dxos/util';
+import { ComplexMap, ComplexSet, exponentialBackoffInterval, SubscriptionGroup } from '@dxos/util';
 
-import { Answer, Message } from '../proto/gen/dxos/mesh/signal';
+import { Answer, SignalMessage } from '../proto/gen/dxos/mesh/signalMessage';
 import { SignalMessaging } from './signal-manager';
 
 interface OfferRecord {
@@ -16,74 +17,166 @@ interface OfferRecord {
 }
 
 interface MessageRouterOptions {
-  onSignal: (message: Message) => Promise<void>;
-  sendMessage: (message: Message) => Promise<void>;
-  onOffer: (message: Message) => Promise<Answer>;
+  sendMessage?: (message: SignalMessage) => Promise<void>;
+  onOffer?: (message: SignalMessage) => Promise<Answer>;
+  onSignal?: (message: SignalMessage) => Promise<void>;
+  retryDelay?: number;
+  timeout?: number;
 }
 
+const log = debug('dxos:network-manager:message-router');
 /**
  * Adds offer/answer RPC and reliable messaging.
  */
 // TODO(mykola): https://github.com/dxos/protocols/issues/1316
 export class MessageRouter implements SignalMessaging {
   private readonly _offerRecords: ComplexMap<PublicKey, OfferRecord> = new ComplexMap(key => key.toHex());
-  private readonly _onSignal: (message: Message) => Promise<void>;
-  private readonly _sendMessage: (message: Message) => Promise<void>;
-  private readonly _onOffer: (message: Message) => Promise<Answer>;
+  private readonly _onSignal: (message: SignalMessage) => Promise<void>;
+  private readonly _sendMessage: (message: SignalMessage) => Promise<void>;
+  private readonly _onOffer: (message: SignalMessage) => Promise<Answer>;
+
+  private readonly _onAckCallbacks = new ComplexMap<PublicKey, () => void>(key => key.toHex());
+  private readonly _receivedMessages = new ComplexSet<PublicKey>(key => key.toHex());
+  private readonly _retryDelay: number;
+  private readonly _timeout: number;
+
+  private readonly _subscriptions = new SubscriptionGroup();
 
   constructor ({
     sendMessage,
     onSignal,
-    onOffer
-  }: MessageRouterOptions) {
+    onOffer,
+    retryDelay = 100,
+    timeout = 3000
+  }: MessageRouterOptions = {}) {
+    assert(sendMessage);
     this._sendMessage = sendMessage;
+    assert(onSignal);
     this._onSignal = onSignal;
+    assert(onOffer);
     this._onOffer = onOffer;
+    this._retryDelay = retryDelay;
+    this._timeout = timeout;
   }
 
-  async receiveMessage (message: Message): Promise<void> {
+  async receiveMessage (message: SignalMessage): Promise<void> {
+    log(`receive message: ${JSON.stringify(message)}`);
+    if (!message.data?.ack) {
+      if (this._receivedMessages.has(message.messageId!)) {
+        return;
+      }
+
+      this._receivedMessages.add(message.messageId!);
+      await this._sendAcknowledgement(message);
+    }
+
     if (message.data?.offer) {
       await this._handleOffer(message);
     } else if (message.data?.answer) {
       await this._resolveAnswers(message);
     } else if (message.data?.signal) {
-      await this._onSignal(message);
+      await this._handleSignal(message);
+    } else if (message.data?.ack) {
+      await this._handleAcknowledgement(message);
     }
   }
 
-  async signal (message: Message): Promise<void> {
+  async signal (message: SignalMessage): Promise<void> {
     assert(message.data?.signal);
-    await this._sendMessage(message);
+    await this._sendReliableMessage(message);
   }
 
-  async offer (message: Message): Promise<Answer> {
+  async offer (message: SignalMessage): Promise<Answer> {
+    message.messageId = PublicKey.random();
     const promise = new Promise<Answer>((resolve, reject) => {
-      assert(message.sessionId);
-      this._offerRecords.set(message.sessionId, { resolve, reject });
+      this._offerRecords.set(message.messageId!, { resolve, reject });
+      return this._sendReliableMessage(message);
     });
-    await this._sendMessage(message);
     return promise;
   }
 
-  private async _resolveAnswers (message: Message): Promise<void> {
-    assert(message.sessionId);
-    const offerRecord = this._offerRecords.get(message.sessionId);
+  private async _sendReliableMessage (message: SignalMessage): Promise<PublicKey> {
+    // Setting unique messageId if it not specified yet.
+    message.messageId = message.messageId ?? PublicKey.random();
+    log(`sent message: ${JSON.stringify(message)}`);
+
+    // Setting retry interval if signal was not acknowledged.
+    const cancelRetry = exponentialBackoffInterval(async () => {
+      log(`retrying message: ${JSON.stringify(message)}`);
+      try {
+        await this._sendMessage(message);
+      } catch (error) {
+        log(`ERROR failed to send message: ${error}`);
+      }
+    }, this._retryDelay);
+
+    const timeout = setTimeout(() => {
+      log('Signal was not delivered!');
+      this._onAckCallbacks.delete(message.messageId!);
+      cancelRetry();
+    }, this._timeout);
+
+    assert(!this._onAckCallbacks.has(message.messageId!));
+    this._onAckCallbacks.set(message.messageId, () => {
+      this._onAckCallbacks.delete(message.messageId!);
+      cancelRetry();
+      clearTimeout(timeout);
+    });
+    this._subscriptions.push(cancelRetry);
+    this._subscriptions.push(() => clearTimeout(timeout));
+
+    await this._sendMessage(message);
+    return message.messageId;
+  }
+
+  private async _resolveAnswers (message: SignalMessage): Promise<void> {
+    assert(message.data?.answer?.offerMessageId, 'No offerMessageId');
+    const offerRecord = this._offerRecords.get(message.data.answer.offerMessageId);
     if (offerRecord) {
-      this._offerRecords.delete(message.sessionId);
-      assert(message.data?.answer);
+      this._offerRecords.delete(message.data.answer.offerMessageId);
+      assert(message.data?.answer, 'No Answer');
+      log(`resolving answer with ${message.data.answer}`);
       offerRecord.resolve(message.data.answer);
     }
   }
 
-  private async _handleOffer (message: Message): Promise<void> {
+  private async _handleOffer (message: SignalMessage): Promise<void> {
     const answer = await this._onOffer(message);
-    const answerMessage = {
+    answer.offerMessageId = message.messageId;
+    const answerMessage: SignalMessage = {
+      id: message.remoteId,
+      remoteId: message.id,
+      topic: message.topic,
+      sessionId: message.sessionId,
+      data: { answer }
+    };
+    await this._sendReliableMessage(answerMessage);
+  }
+
+  private async _handleSignal (message: SignalMessage): Promise<void> {
+    assert(message.messageId);
+    await this._onSignal(message);
+  }
+
+  private async _handleAcknowledgement (message: SignalMessage): Promise<void> {
+    assert(message.data?.ack?.messageId);
+    this._onAckCallbacks.get(message.data.ack.messageId)?.();
+  }
+
+  private async _sendAcknowledgement (message: SignalMessage): Promise<void> {
+    assert(message.messageId);
+    const ackMessage = {
       id: message.remoteId,
       remoteId: message.id,
       topic: message.topic,
       sessionId: message.sessionId,
-      data: { answer: answer }
+      data: { ack: { messageId: message.messageId } }
     };
-    await this._sendMessage(answerMessage);
+    log(`sent ack: ${JSON.stringify(ackMessage)}`);
+    await this._sendMessage(ackMessage);
+  }
+
+  destroy (): void {
+    this._subscriptions.unsubscribe();
   }
 }