@dxos/messaging 0.3.0 → 0.3.1-main.5e68e9d

package/dist/lib/browser/index.mjs

@@ -13,11 +13,9 @@ import { ComplexMap, ComplexSet } from "@dxos/util";
 var MESSAGE_TIMEOUT = 1e4;
 
 // packages/core/mesh/messaging/src/messenger.ts
-var __dxlog_file = "/mnt/ramdisk/work/packages/core/mesh/messaging/src/messenger.ts";
+var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/messenger.ts";
 var ReliablePayload = schema.getCodecForType("dxos.mesh.messaging.ReliablePayload");
 var Acknowledgement = schema.getCodecForType("dxos.mesh.messaging.Acknowledgement");
-var ERROR_LIMIT = 5;
-var NETWORK_REBOOT_DELAY = 3e3;
 var RECEIVED_MESSAGES_GC_INTERVAL = 12e4;
 var Messenger = class {
   constructor({ signalManager, retryDelay = 300 }) {
@@ -32,8 +30,6 @@ var Messenger = class {
     */
     this._toClear = new ComplexSet(PublicKey.hash);
     this._closed = true;
-    this._errorCount = 0;
-    this._rebooting = false;
     this._signalManager = signalManager;
     this._retryDelay = retryDelay;
     this.open();
@@ -47,14 +43,14 @@ var Messenger = class {
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 74,
+      L: 69,
       S: this,
       C: (f, a) => f(...a)
     });
     this._ctx = new Context({
       onError: (err) => log.catch(err, void 0, {
         F: __dxlog_file,
-        L: 76,
+        L: 71,
         S: this,
         C: (f, a) => f(...a)
       })
@@ -64,7 +60,7 @@ var Messenger = class {
         from: message.author
       }, {
         F: __dxlog_file,
-        L: 80,
+        L: 75,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -78,7 +74,7 @@ var Messenger = class {
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 95,
+      L: 90,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -93,7 +89,7 @@ var Messenger = class {
   async sendMessage({ author, recipient, payload }) {
     invariant(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 107,
+      L: 102,
       S: this,
       A: [
         "!this._closed",
@@ -107,7 +103,7 @@ var Messenger = class {
     };
     invariant(!this._onAckCallbacks.has(reliablePayload.messageId), void 0, {
       F: __dxlog_file,
-      L: 114,
+      L: 109,
       S: this,
       A: [
         "!this._onAckCallbacks.has(reliablePayload.messageId!)",
@@ -120,7 +116,7 @@ var Messenger = class {
       recipient
     }, {
       F: __dxlog_file,
-      L: 115,
+      L: 110,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -135,7 +131,7 @@ var Messenger = class {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 129,
+        L: 124,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -147,22 +143,21 @@ var Messenger = class {
         err
       }, {
         F: __dxlog_file,
-        L: 131,
+        L: 126,
         S: this,
         C: (f, a) => f(...a)
       }));
     }, this._retryDelay);
-    scheduleTask(messageContext, async () => {
+    scheduleTask(messageContext, () => {
       log("message not delivered", {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 140,
+        L: 135,
         S: this,
         C: (f, a) => f(...a)
       });
       this._onAckCallbacks.delete(reliablePayload.messageId);
-      await this.errorLimitCheck();
       timeoutHit(new ProtocolTimeoutError("signaling message not delivered", new TimeoutError(MESSAGE_TIMEOUT, "Message not delivered")));
       void messageContext.dispose();
     }, MESSAGE_TIMEOUT);
@@ -178,54 +173,6 @@ var Messenger = class {
     });
     return promise;
   }
-  async errorLimitCheck() {
-    log(`checking error limit ${this._errorCount}`, void 0, {
-      F: __dxlog_file,
-      L: 165,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    if (this._errorCount++ > ERROR_LIMIT) {
-      await this.rebootNetwork();
-    }
-  }
-  async rebootNetwork() {
-    if (this._rebooting) {
-      return;
-    }
-    this._rebooting = true;
-    log("rebooting Messenger/SignalManager", void 0, {
-      F: __dxlog_file,
-      L: 177,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this.close();
-    await this._signalManager.close();
-    log("pausing", void 0, {
-      F: __dxlog_file,
-      L: 180,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await new Promise((resolve) => setTimeout(resolve, NETWORK_REBOOT_DELAY));
-    log("done pausing", void 0, {
-      F: __dxlog_file,
-      L: 182,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    await this.open();
-    await this._signalManager.open();
-    log("done rebooting", void 0, {
-      F: __dxlog_file,
-      L: 185,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    this._errorCount = 0;
-    this._rebooting = false;
-  }
   /**
   * Subscribes onMessage function to messages that contains payload with payloadType.
   * @param payloadType if not specified, onMessage will be subscribed to all types of messages.
@@ -233,7 +180,7 @@ var Messenger = class {
   async listen({ peerId, payloadType, onMessage }) {
     invariant(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 204,
+      L: 171,
       S: this,
       A: [
         "!this._closed",
@@ -297,7 +244,7 @@ var Messenger = class {
   async _handleReliablePayload({ author, recipient, payload }) {
     invariant(payload.type_url === "dxos.mesh.messaging.ReliablePayload", void 0, {
       F: __dxlog_file,
-      L: 265,
+      L: 232,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.ReliablePayload'",
@@ -311,7 +258,7 @@ var Messenger = class {
       messageId: reliablePayload.messageId
     }, {
       F: __dxlog_file,
-      L: 268,
+      L: 235,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -333,7 +280,7 @@ var Messenger = class {
   async _handleAcknowledgement({ payload }) {
     invariant(payload.type_url === "dxos.mesh.messaging.Acknowledgement", void 0, {
       F: __dxlog_file,
-      L: 290,
+      L: 257,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.Acknowledgement'",
@@ -349,7 +296,7 @@ var Messenger = class {
       to: author
     }, {
       F: __dxlog_file,
-      L: 303,
+      L: 270,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -400,7 +347,7 @@ var Messenger = class {
         elapsed
       }, {
         F: __dxlog_file,
-        L: 351,
+        L: 318,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -426,7 +373,7 @@ import { PublicKey as PublicKey2 } from "@dxos/keys";
 import { log as log2 } from "@dxos/log";
 import { schema as schema2, trace as trace2 } from "@dxos/protocols";
 import { createProtoRpcPeer } from "@dxos/rpc";
-var __dxlog_file2 = "/mnt/ramdisk/work/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
 var SignalRPCClient = class {
   constructor({ url, callbacks = {} }) {
     this._connectTrigger = new Trigger();
@@ -661,7 +608,7 @@ var SignalRPCClient = class {
 };
 
 // packages/core/mesh/messaging/src/signal-client/signal-client.ts
-var __dxlog_file3 = "/mnt/ramdisk/work/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
 var DEFAULT_RECONNECT_TIMEOUT = 100;
 var MAX_RECONNECT_TIMEOUT = 5e3;
 var ERROR_RECONCILE_DELAY = 1e3;
@@ -1128,7 +1075,7 @@ import { PublicKey as PublicKey4 } from "@dxos/keys";
 import { log as log4 } from "@dxos/log";
 import { schema as schema3 } from "@dxos/protocols";
 import { ComplexMap as ComplexMap3, ComplexSet as ComplexSet3 } from "@dxos/util";
-var __dxlog_file4 = "/mnt/ramdisk/work/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
 var MemorySignalManagerContext = class {
   constructor() {
     // Swarm messages.
@@ -1384,12 +1331,12 @@ var dec = (payload) => {
 };
 
 // packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts
-import { Event as Event3, synchronized } from "@dxos/async";
+import { Event as Event3, sleep as sleep2, synchronized } from "@dxos/async";
 import { Context as Context4 } from "@dxos/context";
 import { invariant as invariant5 } from "@dxos/invariant";
 import { PublicKey as PublicKey5 } from "@dxos/keys";
 import { log as log5 } from "@dxos/log";
-import { trace as trace4 } from "@dxos/protocols";
+import { RateLimitExceededError, trace as trace4 } from "@dxos/protocols";
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1400,12 +1347,15 @@ function _ts_decorate(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file5 = "/mnt/ramdisk/work/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
+var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
+var MAX_SERVER_FAILURES = 5;
+var WSS_SIGNAL_SERVER_REBOOT_DELAY = 3e3;
 var WebsocketSignalManager = class {
   constructor(_hosts) {
     this._hosts = _hosts;
     this._servers = /* @__PURE__ */ new Map();
     this._opened = false;
+    this.failureCount = /* @__PURE__ */ new Map();
     this.statusChanged = new Event3();
     this.commandTrace = new Event3();
     this.swarmEvent = new Event3();
@@ -1415,7 +1365,7 @@ var WebsocketSignalManager = class {
       hosts: this._hosts
     }, {
       F: __dxlog_file5,
-      L: 43,
+      L: 46,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1426,6 +1376,7 @@ var WebsocketSignalManager = class {
       const server = new SignalClient(host.server, async (message) => this.onMessage.emit(message), async (data) => this.swarmEvent.emit(data));
       server.statusChanged.on(() => this.statusChanged.emit(this.getStatus()));
       this._servers.set(host.server, server);
+      this.failureCount.set(host.server, 0);
       server.commandTrace.on((trace5) => this.commandTrace.emit(trace5));
     }
   }
@@ -1437,7 +1388,7 @@ var WebsocketSignalManager = class {
       hosts: this._hosts
     }, {
       F: __dxlog_file5,
-      L: 65,
+      L: 69,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1445,7 +1396,7 @@ var WebsocketSignalManager = class {
       id: this._instanceId
     }), {
       F: __dxlog_file5,
-      L: 66,
+      L: 70,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1458,7 +1409,7 @@ var WebsocketSignalManager = class {
       id: this._instanceId
     }), {
       F: __dxlog_file5,
-      L: 73,
+      L: 77,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1471,6 +1422,38 @@ var WebsocketSignalManager = class {
     await this._ctx.dispose();
     await Promise.all(Array.from(this._servers.values()).map((server) => server.close()));
   }
+  async restartServer(serverName) {
+    log5("Restarting server", {
+      serverName
+    }, {
+      F: __dxlog_file5,
+      L: 93,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    invariant5(this._opened, "server already closed", {
+      F: __dxlog_file5,
+      L: 94,
+      S: this,
+      A: [
+        "this._opened",
+        "'server already closed'"
+      ]
+    });
+    const server = this._servers.get(serverName);
+    invariant5(server, "server not found", {
+      F: __dxlog_file5,
+      L: 97,
+      S: this,
+      A: [
+        "server",
+        "'server not found'"
+      ]
+    });
+    await server.close();
+    await sleep2(WSS_SIGNAL_SERVER_REBOOT_DELAY);
+    await server.open();
+  }
   getStatus() {
     return Array.from(this._servers.values()).map((server) => server.getStatus());
   }
@@ -1480,13 +1463,13 @@ var WebsocketSignalManager = class {
       peerId
     }, {
       F: __dxlog_file5,
-      L: 93,
+      L: 110,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
       F: __dxlog_file5,
-      L: 94,
+      L: 111,
       S: this,
       A: [
         "this._opened",
@@ -1504,13 +1487,13 @@ var WebsocketSignalManager = class {
       peerId
     }, {
       F: __dxlog_file5,
-      L: 100,
+      L: 117,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
       F: __dxlog_file5,
-      L: 101,
+      L: 118,
       S: this,
       A: [
         "this._opened",
@@ -1523,44 +1506,75 @@ var WebsocketSignalManager = class {
     }));
   }
   async sendMessage({ author, recipient, payload }) {
-    log5(`Signal ${recipient}`, void 0, {
+    log5(`Signal ${recipient.truncate()}`, void 0, {
       F: __dxlog_file5,
-      L: 115,
+      L: 132,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
       F: __dxlog_file5,
-      L: 116,
+      L: 133,
       S: this,
       A: [
         "this._opened",
         "'Closed'"
       ]
     });
-    void this._forEachServer(async (server) => {
+    void this._forEachServer(async (server, serverName) => {
       void server.sendMessage({
         author,
         recipient,
         payload
-      }).catch((err) => log5(err, void 0, {
+      }).catch((err) => {
+        if (err instanceof RateLimitExceededError) {
+          log5("WSS rate limit exceeded", {
+            err
+          }, {
+            F: __dxlog_file5,
+            L: 138,
+            S: this,
+            C: (f, a) => f(...a)
+          });
+        } else {
+          log5(`error sending to ${serverName}`, {
+            err
+          }, {
+            F: __dxlog_file5,
+            L: 140,
+            S: this,
+            C: (f, a) => f(...a)
+          });
+          void this.checkServerFailure(serverName);
+        }
+      });
+    });
+  }
+  async checkServerFailure(serverName) {
+    const failureCount = this.failureCount.get(serverName) ?? 0;
+    if (failureCount > MAX_SERVER_FAILURES) {
+      log5.warn(`Too many failures sending to ${serverName} (${failureCount} > ${MAX_SERVER_FAILURES}), restarting`, void 0, {
         F: __dxlog_file5,
-        L: 119,
+        L: 151,
         S: this,
         C: (f, a) => f(...a)
-      }));
-    });
+      });
+      await this.restartServer(serverName);
+      this.failureCount.set(serverName, 0);
+      return;
+    }
+    this.failureCount.set(serverName, (this.failureCount.get(serverName) ?? 0) + 1);
   }
   async subscribeMessages(peerId) {
     log5(`Subscribed for message stream peerId=${peerId}`, void 0, {
       F: __dxlog_file5,
-      L: 124,
+      L: 160,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
       F: __dxlog_file5,
-      L: 125,
+      L: 161,
       S: this,
       A: [
         "this._opened",
@@ -1572,13 +1586,13 @@ var WebsocketSignalManager = class {
   async unsubscribeMessages(peerId) {
     log5(`Subscribed for message stream peerId=${peerId}`, void 0, {
       F: __dxlog_file5,
-      L: 131,
+      L: 167,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
       F: __dxlog_file5,
-      L: 132,
+      L: 168,
       S: this,
       A: [
         "this._opened",
@@ -1591,25 +1605,31 @@ var WebsocketSignalManager = class {
     this._ctx = new Context4({
       onError: (err) => log5.catch(err, void 0, {
         F: __dxlog_file5,
-        L: 139,
+        L: 175,
         S: this,
         C: (f, a) => f(...a)
       })
     });
   }
   async _forEachServer(fn) {
-    return Promise.all(Array.from(this._servers.values()).map(fn));
+    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server]) => fn(server, serverName)));
   }
 };
 _ts_decorate([
   synchronized
 ], WebsocketSignalManager.prototype, "open", null);
+_ts_decorate([
+  synchronized
+], WebsocketSignalManager.prototype, "close", null);
 _ts_decorate([
   synchronized
 ], WebsocketSignalManager.prototype, "join", null);
 _ts_decorate([
   synchronized
 ], WebsocketSignalManager.prototype, "leave", null);
+_ts_decorate([
+  synchronized
+], WebsocketSignalManager.prototype, "checkServerFailure", null);
 export {
   MemorySignalManager,
   MemorySignalManagerContext,