@dxos/messaging 0.3.0 → 0.3.1-main.0fddbd4

package/src/messenger.ts

@@ -26,9 +26,6 @@ export interface MessengerOptions {
 const ReliablePayload = schema.getCodecForType('dxos.mesh.messaging.ReliablePayload');
 const Acknowledgement = schema.getCodecForType('dxos.mesh.messaging.Acknowledgement');
 
-const ERROR_LIMIT = 5;
-const NETWORK_REBOOT_DELAY = 3_000;
-
 const RECEIVED_MESSAGES_GC_INTERVAL = 120_000;
 
 /**
@@ -56,8 +53,6 @@ export class Messenger {
   private _ctx!: Context;
   private _closed = true;
   private readonly _retryDelay: number;
-  private _errorCount = 0;
-  private _rebooting = false;
 
   constructor({ signalManager, retryDelay = 300 }: MessengerOptions) {
     this._signalManager = signalManager;
@@ -136,10 +131,9 @@ export class Messenger {
 
     scheduleTask(
       messageContext,
-      async () => {
+      () => {
         log('message not delivered', { messageId: reliablePayload.messageId });
         this._onAckCallbacks.delete(reliablePayload.messageId!);
-        await this.errorLimitCheck();
         timeoutHit(
           new ProtocolTimeoutError(
             'signaling message not delivered',
@@ -161,33 +155,6 @@ export class Messenger {
     return promise;
   }
 
-  private async errorLimitCheck() {
-    log(`checking error limit ${this._errorCount}`);
-    if (this._errorCount++ > ERROR_LIMIT) {
-      await this.rebootNetwork();
-    }
-  }
-
-  private async rebootNetwork() {
-    if (this._rebooting) {
-      return;
-    }
-    this._rebooting = true;
-
-    log('rebooting Messenger/SignalManager');
-    await this.close();
-    await this._signalManager.close();
-    log('pausing');
-    await new Promise((resolve) => setTimeout(resolve, NETWORK_REBOOT_DELAY));
-    log('done pausing');
-    await this.open();
-    await this._signalManager.open();
-    log('done rebooting');
-
-    this._errorCount = 0;
-    this._rebooting = false;
-  }
-
   /**
    * Subscribes onMessage function to messages that contains payload with payloadType.
    * @param payloadType if not specified, onMessage will be subscribed to all types of messages.

package/src/signal-manager/websocket-signal-manager.ts

@@ -2,19 +2,21 @@
 // Copyright 2020 DXOS.org
 //
 
-import { Event, synchronized } from '@dxos/async';
+import { Event, sleep, synchronized } from '@dxos/async';
 import { Any } from '@dxos/codec-protobuf';
 import { Context } from '@dxos/context';
 import { invariant } from '@dxos/invariant';
 import { PublicKey } from '@dxos/keys';
 import { log } from '@dxos/log';
-import { trace } from '@dxos/protocols';
+import { RateLimitExceededError, trace } from '@dxos/protocols';
 import { Runtime } from '@dxos/protocols/proto/dxos/config';
 import { SwarmEvent } from '@dxos/protocols/proto/dxos/mesh/signal';
 
 import { SignalManager } from './signal-manager';
 import { CommandTrace, SignalClient, SignalStatus } from '../signal-client';
 
+const MAX_SERVER_FAILURES = 5;
+const WSS_SIGNAL_SERVER_REBOOT_DELAY = 3_000;
 /**
  * Manages connection to multiple Signal Servers over WebSocket
  */
@@ -24,6 +26,7 @@ export class WebsocketSignalManager implements SignalManager {
   private _ctx!: Context;
   private _opened = false;
 
+  readonly failureCount = new Map<string, number>();
   readonly statusChanged = new Event<SignalStatus[]>();
   readonly commandTrace = new Event<CommandTrace>();
   readonly swarmEvent = new Event<{
@@ -53,6 +56,7 @@ export class WebsocketSignalManager implements SignalManager {
       server.statusChanged.on(() => this.statusChanged.emit(this.getStatus()));
 
       this._servers.set(host.server, server);
+      this.failureCount.set(host.server, 0);
       server.commandTrace.on((trace) => this.commandTrace.emit(trace));
     }
   }
@@ -73,6 +77,7 @@ export class WebsocketSignalManager implements SignalManager {
     log.trace('dxos.mesh.websocket-signal-manager.open', trace.end({ id: this._instanceId }));
   }
 
+  @synchronized
   async close() {
     if (!this._opened) {
       return;
@@ -84,6 +89,18 @@ export class WebsocketSignalManager implements SignalManager {
     await Promise.all(Array.from(this._servers.values()).map((server) => server.close()));
   }
 
+  async restartServer(serverName: string) {
+    log('Restarting server', { serverName });
+    invariant(this._opened, 'server already closed');
+
+    const server = this._servers.get(serverName);
+    invariant(server, 'server not found');
+
+    await server.close();
+    await sleep(WSS_SIGNAL_SERVER_REBOOT_DELAY);
+    await server.open();
+  }
+
   getStatus(): SignalStatus[] {
     return Array.from(this._servers.values()).map((server) => server.getStatus());
   }
@@ -112,14 +129,33 @@ export class WebsocketSignalManager implements SignalManager {
     recipient: PublicKey;
     payload: Any;
   }): Promise<void> {
-    log(`Signal ${recipient}`);
+    log(`Signal ${recipient.truncate()}`);
     invariant(this._opened, 'Closed');
 
-    void this._forEachServer(async (server) => {
-      void server.sendMessage({ author, recipient, payload }).catch((err) => log(err));
+    void this._forEachServer(async (server, serverName) => {
+      void server.sendMessage({ author, recipient, payload }).catch((err) => {
+        if (err instanceof RateLimitExceededError) {
+          log('WSS rate limit exceeded', { err });
+        } else {
+          log(`error sending to ${serverName}`, { err });
+          void this.checkServerFailure(serverName);
+        }
+      });
     });
   }
 
+  @synchronized
+  async checkServerFailure(serverName: string) {
+    const failureCount = this.failureCount.get(serverName!) ?? 0;
+    if (failureCount > MAX_SERVER_FAILURES) {
+      log.warn(`Too many failures sending to ${serverName} (${failureCount} > ${MAX_SERVER_FAILURES}), restarting`);
+      await this.restartServer(serverName!);
+      this.failureCount.set(serverName!, 0);
+      return;
+    }
+    this.failureCount.set(serverName!, (this.failureCount.get(serverName!) ?? 0) + 1);
+  }
+
   async subscribeMessages(peerId: PublicKey) {
     log(`Subscribed for message stream peerId=${peerId}`);
     invariant(this._opened, 'Closed');
@@ -140,7 +176,9 @@ export class WebsocketSignalManager implements SignalManager {
     });
   }
 
-  private async _forEachServer<ReturnType>(fn: (server: SignalClient) => Promise<ReturnType>): Promise<ReturnType[]> {
-    return Promise.all(Array.from(this._servers.values()).map(fn));
+  private async _forEachServer<ReturnType>(
+    fn: (server: SignalClient, serverName: string) => Promise<ReturnType>,
+  ): Promise<ReturnType[]> {
+    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server]) => fn(server, serverName)));
   }
 }