@dxos/messaging 0.5.9-main.ea1d25b → 0.5.9-main.f099efe

package/dist/lib/node/index.cjs

@@ -43,6 +43,7 @@ var import_keys = require("@dxos/keys");
 var import_log = require("@dxos/log");
 var import_protocols = require("@dxos/protocols");
 var import_util = require("@dxos/util");
+var import_tracing = require("@dxos/tracing");
 var import_async2 = require("@dxos/async");
 var import_context2 = require("@dxos/context");
 var import_invariant2 = require("@dxos/invariant");
@@ -50,31 +51,52 @@ var import_keys2 = require("@dxos/keys");
 var import_log2 = require("@dxos/log");
 var import_protocols2 = require("@dxos/protocols");
 var import_signal = require("@dxos/protocols/proto/dxos/mesh/signal");
-var import_util2 = require("@dxos/util");
-var import_isomorphic_ws = __toESM(require("isomorphic-ws"));
+var import_tracing2 = require("@dxos/tracing");
 var import_async3 = require("@dxos/async");
 var import_context3 = require("@dxos/context");
-var import_invariant3 = require("@dxos/invariant");
 var import_keys3 = require("@dxos/keys");
 var import_log3 = require("@dxos/log");
-var import_protocols3 = require("@dxos/protocols");
-var import_rpc = require("@dxos/rpc");
+var import_util2 = require("@dxos/util");
+var import_isomorphic_ws = __toESM(require("isomorphic-ws"));
 var import_async4 = require("@dxos/async");
 var import_context4 = require("@dxos/context");
-var import_invariant4 = require("@dxos/invariant");
+var import_invariant3 = require("@dxos/invariant");
 var import_keys4 = require("@dxos/keys");
 var import_log4 = require("@dxos/log");
-var import_protocols4 = require("@dxos/protocols");
-var import_util3 = require("@dxos/util");
+var import_protocols3 = require("@dxos/protocols");
+var import_rpc = require("@dxos/rpc");
+var import_tracing3 = require("@dxos/tracing");
 var import_async5 = require("@dxos/async");
 var import_context5 = require("@dxos/context");
-var import_invariant5 = require("@dxos/invariant");
+var import_invariant4 = require("@dxos/invariant");
 var import_keys5 = require("@dxos/keys");
 var import_log5 = require("@dxos/log");
+var import_protocols4 = require("@dxos/protocols");
+var import_util3 = require("@dxos/util");
+var import_async6 = require("@dxos/async");
+var import_context6 = require("@dxos/context");
+var import_invariant5 = require("@dxos/invariant");
+var import_keys6 = require("@dxos/keys");
+var import_log6 = require("@dxos/log");
 var import_protocols5 = require("@dxos/protocols");
+var import_util4 = require("@dxos/util");
+var import_tracing4 = require("@dxos/tracing");
 var import_invariant6 = require("@dxos/invariant");
-var import_log6 = require("@dxos/log");
+var import_log7 = require("@dxos/log");
 var import_services = require("@dxos/protocols/proto/dxos/client/services");
+var MessengerMonitor = class {
+  recordMessageAckFailed() {
+    import_tracing.trace.metrics.increment("mesh.signal.messenger.failed-ack", 1);
+  }
+  recordReliableMessage(params) {
+    import_tracing.trace.metrics.increment("mesh.signal.messenger.reliable-send", 1, {
+      tags: {
+        success: params.sent,
+        attempts: params.sendAttempts
+      }
+    });
+  }
+};
 var MESSAGE_TIMEOUT = 1e4;
 var __dxlog_file = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/messenger.ts";
 var ReliablePayload = import_protocols.schema.getCodecForType("dxos.mesh.messaging.ReliablePayload");
@@ -82,6 +104,7 @@ var Acknowledgement = import_protocols.schema.getCodecForType("dxos.mesh.messagi
 var RECEIVED_MESSAGES_GC_INTERVAL = 12e4;
 var Messenger = class {
   constructor({ signalManager, retryDelay = 300 }) {
+    this._monitor = new MessengerMonitor();
     this._listeners = new import_util.ComplexMap(({ peerId, payloadType }) => peerId.toHex() + payloadType);
     this._defaultListeners = new import_util.ComplexMap(import_keys.PublicKey.hash);
     this._onAckCallbacks = new import_util.ComplexMap(import_keys.PublicKey.hash);
@@ -101,24 +124,27 @@ var Messenger = class {
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 69,
+      L: 71,
       S: this,
       C: (f, a) => f(...a)
     });
     this._ctx = new import_context.Context({
       onError: (err) => import_log.log.catch(err, void 0, {
         F: __dxlog_file,
-        L: 71,
+        L: 73,
         S: this,
         C: (f, a) => f(...a)
       })
+    }, {
+      F: __dxlog_file,
+      L: 72
     });
     this._ctx.onDispose(this._signalManager.onMessage.on(async (message) => {
       (0, import_log.log)("received message", {
         from: message.author
       }, {
         F: __dxlog_file,
-        L: 75,
+        L: 77,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -132,7 +158,7 @@ var Messenger = class {
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 90,
+      L: 92,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -147,7 +173,7 @@ var Messenger = class {
   async sendMessage({ author, recipient, payload }) {
     (0, import_invariant.invariant)(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 102,
+      L: 104,
       S: this,
       A: [
         "!this._closed",
@@ -161,7 +187,7 @@ var Messenger = class {
     };
     (0, import_invariant.invariant)(!this._onAckCallbacks.has(reliablePayload.messageId), void 0, {
       F: __dxlog_file,
-      L: 109,
+      L: 111,
       S: this,
       A: [
         "!this._onAckCallbacks.has(reliablePayload.messageId!)",
@@ -174,12 +200,13 @@ var Messenger = class {
       recipient
     }, {
       F: __dxlog_file,
-      L: 110,
+      L: 112,
       S: this,
       C: (f, a) => f(...a)
     });
     let messageReceived;
     let timeoutHit;
+    let sendAttempts = 0;
     const promise = new Promise((resolve, reject) => {
       messageReceived = resolve;
       timeoutHit = reject;
@@ -189,10 +216,11 @@ var Messenger = class {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 124,
+        L: 127,
         S: this,
         C: (f, a) => f(...a)
       });
+      sendAttempts++;
       await this._encodeAndSend({
         author,
         recipient,
@@ -201,7 +229,7 @@ var Messenger = class {
         err
       }, {
         F: __dxlog_file,
-        L: 126,
+        L: 130,
         S: this,
         C: (f, a) => f(...a)
       }));
@@ -211,18 +239,26 @@ var Messenger = class {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 135,
+        L: 139,
         S: this,
         C: (f, a) => f(...a)
       });
       this._onAckCallbacks.delete(reliablePayload.messageId);
       timeoutHit(new import_protocols.TimeoutError("signaling message not delivered", new import_async.TimeoutError(MESSAGE_TIMEOUT, "Message not delivered")));
       void messageContext.dispose();
+      this._monitor.recordReliableMessage({
+        sendAttempts,
+        sent: false
+      });
     }, MESSAGE_TIMEOUT);
     this._onAckCallbacks.set(reliablePayload.messageId, () => {
       messageReceived();
       this._onAckCallbacks.delete(reliablePayload.messageId);
       void messageContext.dispose();
+      this._monitor.recordReliableMessage({
+        sendAttempts,
+        sent: true
+      });
     });
     await this._encodeAndSend({
       author,
@@ -238,7 +274,7 @@ var Messenger = class {
   async listen({ peerId, payloadType, onMessage }) {
     (0, import_invariant.invariant)(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 171,
+      L: 177,
       S: this,
       A: [
         "!this._closed",
@@ -302,7 +338,7 @@ var Messenger = class {
   async _handleReliablePayload({ author, recipient, payload }) {
     (0, import_invariant.invariant)(payload.type_url === "dxos.mesh.messaging.ReliablePayload", void 0, {
       F: __dxlog_file,
-      L: 232,
+      L: 238,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.ReliablePayload'",
@@ -316,15 +352,20 @@ var Messenger = class {
       messageId: reliablePayload.messageId
     }, {
       F: __dxlog_file,
-      L: 235,
+      L: 241,
       S: this,
       C: (f, a) => f(...a)
     });
-    await this._sendAcknowledgement({
-      author,
-      recipient,
-      messageId: reliablePayload.messageId
-    });
+    try {
+      await this._sendAcknowledgement({
+        author,
+        recipient,
+        messageId: reliablePayload.messageId
+      });
+    } catch (err) {
+      this._monitor.recordMessageAckFailed();
+      throw err;
+    }
     if (this._receivedMessages.has(reliablePayload.messageId)) {
       return;
     }
@@ -338,7 +379,7 @@ var Messenger = class {
   async _handleAcknowledgement({ payload }) {
     (0, import_invariant.invariant)(payload.type_url === "dxos.mesh.messaging.Acknowledgement", void 0, {
       F: __dxlog_file,
-      L: 258,
+      L: 269,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.Acknowledgement'",
@@ -354,7 +395,7 @@ var Messenger = class {
       to: author
     }, {
       F: __dxlog_file,
-      L: 271,
+      L: 282,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -405,26 +446,279 @@ var Messenger = class {
         elapsed
       }, {
         F: __dxlog_file,
-        L: 319,
+        L: 330,
         S: this,
         C: (f, a) => f(...a)
       });
     }
   }
 };
-var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
+var SignalClientMonitor = class {
+  constructor() {
+    this._performance = {
+      sentMessages: 0,
+      receivedMessages: 0,
+      reconnectCounter: 0,
+      joinCounter: 0,
+      leaveCounter: 0
+    };
+    this._connectionStarted = /* @__PURE__ */ new Date();
+    this._lastStateChange = /* @__PURE__ */ new Date();
+  }
+  getRecordedTimestamps() {
+    return {
+      connectionStarted: this._connectionStarted,
+      lastStateChange: this._lastStateChange
+    };
+  }
+  recordStateChangeTime() {
+    this._lastStateChange = /* @__PURE__ */ new Date();
+  }
+  recordConnectionStartTime() {
+    this._connectionStarted = /* @__PURE__ */ new Date();
+  }
+  recordReconnect(params) {
+    this._performance.reconnectCounter++;
+    import_tracing2.trace.metrics.increment("mesh.signal.signal-client.reconnect", 1, {
+      tags: {
+        success: params.success
+      }
+    });
+  }
+  recordJoin() {
+    this._performance.joinCounter++;
+  }
+  recordLeave() {
+    this._performance.leaveCounter++;
+  }
+  recordMessageReceived(message) {
+    this._performance.receivedMessages++;
+    import_tracing2.trace.metrics.increment("mesh.signal.signal-client.received-total", 1, {
+      tags: createIdentityTags(message)
+    });
+    import_tracing2.trace.metrics.distribution("mesh.signal.signal-client.bytes-in", getByteCount(message), {
+      tags: createIdentityTags(message)
+    });
+  }
+  async recordMessageSending(message, sendMessage) {
+    this._performance.sentMessages++;
+    const tags = createIdentityTags(message);
+    let success = true;
+    try {
+      const reqStart = Date.now();
+      await sendMessage();
+      const reqDuration = Date.now() - reqStart;
+      import_tracing2.trace.metrics.distribution("mesh.signal.signal-client.send-duration", reqDuration, {
+        tags
+      });
+      import_tracing2.trace.metrics.distribution("mesh.signal.signal-client.bytes-out", getByteCount(message), {
+        tags
+      });
+    } catch (err) {
+      success = false;
+    }
+    import_tracing2.trace.metrics.increment("mesh.signal.signal-client.sent-total", 1, {
+      tags: {
+        ...tags,
+        success
+      }
+    });
+  }
+  recordStreamCloseErrors(count) {
+    import_tracing2.trace.metrics.increment("mesh.signal.signal-client.stream-close-errors", count);
+  }
+  recordReconciliation(params) {
+    import_tracing2.trace.metrics.increment("mesh.signal.signal-client.reconciliation", 1, {
+      tags: {
+        success: params.success
+      }
+    });
+  }
+};
+var getByteCount = (message) => {
+  return message.author.asUint8Array().length + message.recipient.asUint8Array().length + message.payload.type_url.length + message.payload.value.length;
+};
+var createIdentityTags = (message) => {
+  return {
+    peer: message.author.toHex()
+  };
+};
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-local-state.ts";
+var SignalLocalState = class {
+  constructor(_onMessage, _onSwarmEvent) {
+    this._onMessage = _onMessage;
+    this._onSwarmEvent = _onSwarmEvent;
+    this._swarmStreams = new import_util2.ComplexMap(({ topic, peerId }) => topic.toHex() + peerId.toHex());
+    this._joinedTopics = new import_util2.ComplexSet(({ topic, peerId }) => topic.toHex() + peerId.toHex());
+    this._subscribedMessages = new import_util2.ComplexSet(({ peerId }) => peerId.toHex());
+    this.messageStreams = new import_util2.ComplexMap((key) => key.toHex());
+    this.reconciled = new import_async3.Event();
+  }
+  async safeCloseStreams() {
+    const streams = [
+      ...this._swarmStreams.values()
+    ].concat([
+      ...this.messageStreams.values()
+    ]);
+    this._swarmStreams.clear();
+    this.messageStreams.clear();
+    const failureCount = (await (0, import_util2.safeAwaitAll)(streams, (s) => s.close())).length;
+    return {
+      failureCount
+    };
+  }
+  join({ topic, peerId }) {
+    this._joinedTopics.add({
+      topic,
+      peerId
+    });
+  }
+  leave({ topic, peerId }) {
+    void this._swarmStreams.get({
+      topic,
+      peerId
+    })?.close();
+    this._swarmStreams.delete({
+      topic,
+      peerId
+    });
+    this._joinedTopics.delete({
+      topic,
+      peerId
+    });
+  }
+  subscribeMessages(peerId) {
+    this._subscribedMessages.add({
+      peerId
+    });
+  }
+  unsubscribeMessages(peerId) {
+    (0, import_log3.log)("unsubscribing from messages", {
+      peerId
+    }, {
+      F: __dxlog_file2,
+      L: 76,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    this._subscribedMessages.delete({
+      peerId
+    });
+    void this.messageStreams.get(peerId)?.close();
+    this.messageStreams.delete(peerId);
+  }
+  async reconcile(ctx, client) {
+    await this._reconcileSwarmSubscriptions(ctx, client);
+    await this._reconcileMessageSubscriptions(ctx, client);
+    this.reconciled.emit();
+  }
+  async _reconcileSwarmSubscriptions(ctx, client) {
+    for (const { topic, peerId } of this._swarmStreams.keys()) {
+      if (this._joinedTopics.has({
+        topic,
+        peerId
+      })) {
+        continue;
+      }
+      void this._swarmStreams.get({
+        topic,
+        peerId
+      })?.close();
+      this._swarmStreams.delete({
+        topic,
+        peerId
+      });
+    }
+    for (const { topic, peerId } of this._joinedTopics.values()) {
+      if (this._swarmStreams.has({
+        topic,
+        peerId
+      })) {
+        continue;
+      }
+      const swarmStream = await (0, import_async3.asyncTimeout)((0, import_context3.cancelWithContext)(ctx, client.join({
+        topic,
+        peerId
+      })), 5e3);
+      swarmStream.subscribe(async (swarmEvent) => {
+        if (this._joinedTopics.has({
+          topic,
+          peerId
+        })) {
+          (0, import_log3.log)("swarm event", {
+            swarmEvent
+          }, {
+            F: __dxlog_file2,
+            L: 112,
+            S: this,
+            C: (f, a) => f(...a)
+          });
+          await this._onSwarmEvent({
+            topic,
+            swarmEvent
+          });
+        }
+      });
+      this._swarmStreams.set({
+        topic,
+        peerId
+      }, swarmStream);
+    }
+  }
+  async _reconcileMessageSubscriptions(ctx, client) {
+    for (const peerId of this.messageStreams.keys()) {
+      if (this._subscribedMessages.has({
+        peerId
+      })) {
+        continue;
+      }
+      void this.messageStreams.get(peerId)?.close();
+      this.messageStreams.delete(peerId);
+    }
+    for (const { peerId } of this._subscribedMessages.values()) {
+      if (this.messageStreams.has(peerId)) {
+        continue;
+      }
+      const messageStream = await (0, import_async3.asyncTimeout)((0, import_context3.cancelWithContext)(ctx, client.receiveMessages(peerId)), 5e3);
+      messageStream.subscribe(async (signalMessage) => {
+        if (this._subscribedMessages.has({
+          peerId
+        })) {
+          const message = {
+            author: import_keys3.PublicKey.from(signalMessage.author),
+            recipient: import_keys3.PublicKey.from(signalMessage.recipient),
+            payload: signalMessage.payload
+          };
+          await this._onMessage(message);
+        }
+      });
+      this.messageStreams.set(peerId, messageStream);
+    }
+  }
+};
+var SignalRpcClientMonitor = class {
+  recordClientCloseFailure(params) {
+    import_tracing3.trace.metrics.increment("mesh.signal.signal-rpc-client.close-failure", 1, {
+      tags: {
+        reason: params.failureReason
+      }
+    });
+  }
+};
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
 var SIGNAL_KEEPALIVE_INTERVAL = 1e4;
 var SignalRPCClient = class {
   constructor({ url, callbacks = {} }) {
-    this._connectTrigger = new import_async3.Trigger();
+    this._connectTrigger = new import_async4.Trigger();
     this._closed = false;
-    this._closeComplete = new import_async3.Trigger();
-    const traceId = import_keys3.PublicKey.random().toHex();
-    import_log3.log.trace("dxos.mesh.signal-rpc-client.constructor", import_protocols3.trace.begin({
+    this._closeComplete = new import_async4.Trigger();
+    this._monitor = new SignalRpcClientMonitor();
+    const traceId = import_keys4.PublicKey.random().toHex();
+    import_log4.log.trace("dxos.mesh.signal-rpc-client.constructor", import_protocols3.trace.begin({
       id: traceId
     }), {
-      F: __dxlog_file2,
-      L: 55,
+      F: __dxlog_file3,
+      L: 60,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -444,9 +738,9 @@ var SignalRPCClient = class {
           try {
             this._socket.send(msg);
           } catch (err) {
-            import_log3.log.warn("send error", err, {
-              F: __dxlog_file2,
-              L: 74,
+            import_log4.log.warn("send error", err, {
+              F: __dxlog_file3,
+              L: 79,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -469,26 +763,35 @@ var SignalRPCClient = class {
     this._socket.onopen = async () => {
       try {
         await this._rpc.open();
-        (0, import_log3.log)(`RPC open ${this._url}`, void 0, {
-          F: __dxlog_file2,
-          L: 95,
+        if (this._closed) {
+          await this._safeCloseRpc();
+          return;
+        }
+        (0, import_log4.log)(`RPC open ${this._url}`, void 0, {
+          F: __dxlog_file3,
+          L: 104,
           S: this,
           C: (f, a) => f(...a)
         });
         this._callbacks.onConnected?.();
         this._connectTrigger.wake();
-        this._keepaliveCtx = new import_context3.Context();
-        (0, import_async3.scheduleTaskInterval)(this._keepaliveCtx, async () => {
+        this._keepaliveCtx = new import_context4.Context(void 0, {
+          F: __dxlog_file3,
+          L: 107
+        });
+        (0, import_async4.scheduleTaskInterval)(this._keepaliveCtx, async () => {
           this._socket?.send("__ping__");
         }, SIGNAL_KEEPALIVE_INTERVAL);
       } catch (err) {
         this._callbacks.onError?.(err);
+        this._socket.close();
+        this._closed = true;
       }
     };
     this._socket.onclose = async () => {
-      (0, import_log3.log)(`Disconnected ${this._url}`, void 0, {
-        F: __dxlog_file2,
-        L: 116,
+      (0, import_log4.log)(`Disconnected ${this._url}`, void 0, {
+        F: __dxlog_file3,
+        L: 127,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -498,89 +801,73 @@ var SignalRPCClient = class {
     };
     this._socket.onerror = async (event) => {
       if (this._closed) {
+        this._socket.close();
         return;
       }
-      this._callbacks.onError?.(event.error ?? new Error(event.message));
-      this._connectTrigger.reset();
-      try {
-        await this._rpc?.close();
-      } catch (err) {
-        import_log3.log.catch(err, void 0, {
-          F: __dxlog_file2,
-          L: 134,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-      }
       this._closed = true;
-      import_log3.log.warn(event.message ?? "Socket error", {
+      this._callbacks.onError?.(event.error ?? new Error(event.message));
+      await this._safeCloseRpc();
+      import_log4.log.warn(`Socket ${event.type ?? "unknown"} error`, {
+        message: event.message,
         url: this._url
       }, {
-        F: __dxlog_file2,
-        L: 138,
+        F: __dxlog_file3,
+        L: 143,
         S: this,
         C: (f, a) => f(...a)
       });
     };
-    import_log3.log.trace("dxos.mesh.signal-rpc-client.constructor", import_protocols3.trace.end({
+    import_log4.log.trace("dxos.mesh.signal-rpc-client.constructor", import_protocols3.trace.end({
       id: traceId
     }), {
-      F: __dxlog_file2,
-      L: 141,
+      F: __dxlog_file3,
+      L: 146,
       S: this,
       C: (f, a) => f(...a)
     });
   }
   async close() {
-    await this._keepaliveCtx?.dispose();
+    if (this._closed) {
+      return;
+    }
     this._closed = true;
+    await this._keepaliveCtx?.dispose();
     try {
-      await this._rpc?.close();
-      if (this._socket?.readyState === import_isomorphic_ws.default.OPEN || this._socket?.readyState === import_isomorphic_ws.default.CONNECTING) {
+      await this._safeCloseRpc();
+      if (this._socket.readyState === import_isomorphic_ws.default.OPEN || this._socket.readyState === import_isomorphic_ws.default.CONNECTING) {
         this._socket.close();
       }
       await this._closeComplete.wait({
         timeout: 1e3
       });
     } catch (err) {
-      import_log3.log.warn("close error", err, {
-        F: __dxlog_file2,
-        L: 156,
-        S: this,
-        C: (f, a) => f(...a)
+      const failureReason = err instanceof import_async4.TimeoutError ? "timeout" : err?.constructor?.name ?? "unknown";
+      this._monitor.recordClientCloseFailure({
+        failureReason
       });
     }
   }
   async join({ topic, peerId }) {
-    (0, import_log3.log)("join", {
+    (0, import_log4.log)("join", {
       topic,
       peerId,
       metadata: this._callbacks?.getMetadata?.()
     }, {
-      F: __dxlog_file2,
-      L: 161,
+      F: __dxlog_file3,
+      L: 172,
       S: this,
       C: (f, a) => f(...a)
     });
-    await this._connectTrigger.wait();
     (0, import_invariant3.invariant)(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 163,
+      F: __dxlog_file3,
+      L: 173,
       S: this,
       A: [
         "!this._closed",
         "'SignalRPCClient is closed'"
       ]
     });
-    (0, import_invariant3.invariant)(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 164,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
+    await this._connectTrigger.wait();
     const swarmStream = this._rpc.rpc.Signal.join({
       swarm: topic.asUint8Array(),
       peer: peerId.asUint8Array(),
@@ -590,17 +877,17 @@ var SignalRPCClient = class {
     return swarmStream;
   }
   async receiveMessages(peerId) {
-    (0, import_log3.log)("receiveMessages", {
+    (0, import_log4.log)("receiveMessages", {
       peerId
     }, {
-      F: __dxlog_file2,
-      L: 175,
+      F: __dxlog_file3,
+      L: 185,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant3.invariant)(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 176,
+      F: __dxlog_file3,
+      L: 186,
       S: this,
       A: [
         "!this._closed",
@@ -608,15 +895,6 @@ var SignalRPCClient = class {
       ]
     });
     await this._connectTrigger.wait();
-    (0, import_invariant3.invariant)(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 178,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
     const messageStream = this._rpc.rpc.Signal.receiveMessages({
       peer: peerId.asUint8Array()
     });
@@ -624,20 +902,20 @@ var SignalRPCClient = class {
     return messageStream;
   }
   async sendMessage({ author, recipient, payload }) {
-    (0, import_log3.log)("sendMessage", {
+    (0, import_log4.log)("sendMessage", {
       author,
       recipient,
       payload,
       metadata: this._callbacks?.getMetadata?.()
     }, {
-      F: __dxlog_file2,
-      L: 187,
+      F: __dxlog_file3,
+      L: 196,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant3.invariant)(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 188,
+      F: __dxlog_file3,
+      L: 197,
       S: this,
       A: [
         "!this._closed",
@@ -645,15 +923,6 @@ var SignalRPCClient = class {
       ]
     });
     await this._connectTrigger.wait();
-    (0, import_invariant3.invariant)(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 190,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
     await this._rpc.rpc.Signal.sendMessage({
       author: author.asUint8Array(),
       recipient: recipient.asUint8Array(),
@@ -661,51 +930,57 @@ var SignalRPCClient = class {
       metadata: this._callbacks?.getMetadata?.()
     });
   }
+  async _safeCloseRpc() {
+    try {
+      this._connectTrigger.reset();
+      await this._rpc.close();
+    } catch (err) {
+      import_log4.log.catch(err, void 0, {
+        F: __dxlog_file3,
+        L: 212,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
+  }
 };
-var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
 var DEFAULT_RECONNECT_TIMEOUT = 100;
 var MAX_RECONNECT_TIMEOUT = 5e3;
 var ERROR_RECONCILE_DELAY = 1e3;
 var RECONCILE_INTERVAL = 5e3;
-var SignalClient = class {
+var SignalClient = class extends import_context2.Resource {
   /**
   * @param _host Signal server websocket URL.
+  * @param onMessage called when a new message is received.
+  * @param onSwarmEvent called when a new swarm event is received.
+  * @param _getMetadata signal-message metadata provider, called for every message.
   */
-  constructor(_host, _onMessage, _onSwarmEvent, _getMetadata) {
+  constructor(_host, onMessage, onSwarmEvent, _getMetadata) {
+    super();
     this._host = _host;
-    this._onMessage = _onMessage;
-    this._onSwarmEvent = _onSwarmEvent;
     this._getMetadata = _getMetadata;
+    this._monitor = new SignalClientMonitor();
     this._state = import_signal.SignalState.CLOSED;
-    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
-    this._connectionStarted = /* @__PURE__ */ new Date();
-    this._lastStateChange = /* @__PURE__ */ new Date();
+    this._lastReconciliationFailed = false;
     this._clientReady = new import_async2.Trigger();
-    this.statusChanged = new import_async2.Event();
-    this.commandTrace = new import_async2.Event();
-    this._swarmStreams = new import_util2.ComplexMap(({ topic, peerId }) => topic.toHex() + peerId.toHex());
-    this._joinedTopics = new import_util2.ComplexSet(({ topic, peerId }) => topic.toHex() + peerId.toHex());
-    this._messageStreams = new import_util2.ComplexMap((key) => key.toHex());
-    this._subscribedMessages = new import_util2.ComplexSet(({ peerId }) => peerId.toHex());
-    this._reconciled = new import_async2.Event();
+    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
     this._instanceId = import_keys2.PublicKey.random().toHex();
-    this._performance = {
-      sentMessages: 0,
-      receivedMessages: 0,
-      reconnectCounter: 0,
-      joinCounter: 0,
-      leaveCounter: 0
-    };
+    this.statusChanged = new import_async2.Event();
     if (!this._host.startsWith("wss://") && !this._host.startsWith("ws://")) {
       throw new Error(`Signal server requires a websocket URL. Provided: ${this._host}`);
     }
+    this.localState = new SignalLocalState((message) => {
+      this._monitor.recordMessageReceived(message);
+      return onMessage(message);
+    }, onSwarmEvent);
   }
-  async open() {
+  async _open() {
     import_log2.log.trace("dxos.mesh.signal-client.open", import_protocols2.trace.begin({
       id: this._instanceId
     }), {
-      F: __dxlog_file3,
-      L: 128,
+      F: __dxlog_file4,
+      L: 83,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -715,26 +990,33 @@ var SignalClient = class {
     ].includes(this._state)) {
       return;
     }
-    this._ctx = new import_context2.Context({
-      onError: (err) => {
-        if (this._state === import_signal.SignalState.CLOSED || this._ctx?.disposed) {
-          return;
-        }
-        if (this._state === import_signal.SignalState.CONNECTED) {
-          import_log2.log.warn("SignalClient error:", err, {
-            F: __dxlog_file3,
-            L: 140,
-            S: this,
-            C: (f, a) => f(...a)
-          });
-        }
-        this._scheduleReconcileAfterError();
-      }
-    });
+    this._setState(import_signal.SignalState.CONNECTING);
     this._reconcileTask = new import_async2.DeferredTask(this._ctx, async () => {
-      await this._reconcileSwarmSubscriptions();
-      await this._reconcileMessageSubscriptions();
-      this._reconciled.emit();
+      try {
+        await (0, import_context2.cancelWithContext)(this._connectionCtx, this._clientReady.wait({
+          timeout: 5e3
+        }));
+        (0, import_invariant2.invariant)(this._state === import_signal.SignalState.CONNECTED, "Not connected to Signal Server", {
+          F: __dxlog_file4,
+          L: 93,
+          S: this,
+          A: [
+            "this._state === SignalState.CONNECTED",
+            "'Not connected to Signal Server'"
+          ]
+        });
+        await this.localState.reconcile(this._connectionCtx, this._client);
+        this._monitor.recordReconciliation({
+          success: true
+        });
+        this._lastReconciliationFailed = false;
+      } catch (err) {
+        this._lastReconciliationFailed = true;
+        this._monitor.recordReconciliation({
+          success: false
+        });
+        throw err;
+      }
     });
     (0, import_async2.scheduleTaskInterval)(this._ctx, async () => {
       if (this._state === import_signal.SignalState.CONNECTED) {
@@ -742,23 +1024,46 @@ var SignalClient = class {
       }
     }, RECONCILE_INTERVAL);
     this._reconnectTask = new import_async2.DeferredTask(this._ctx, async () => {
-      await this._reconnect();
+      try {
+        await this._reconnect();
+        this._monitor.recordReconnect({
+          success: true
+        });
+      } catch (err) {
+        this._monitor.recordReconnect({
+          success: false
+        });
+        throw err;
+      }
     });
-    this._setState(import_signal.SignalState.CONNECTING);
     this._createClient();
     import_log2.log.trace("dxos.mesh.signal-client.open", import_protocols2.trace.end({
       id: this._instanceId
     }), {
-      F: __dxlog_file3,
-      L: 169,
+      F: __dxlog_file4,
+      L: 126,
       S: this,
       C: (f, a) => f(...a)
     });
   }
-  async close() {
+  async _catch(err) {
+    if (this._state === import_signal.SignalState.CLOSED || this._ctx.disposed) {
+      return;
+    }
+    if (this._state === import_signal.SignalState.CONNECTED && !this._lastReconciliationFailed) {
+      import_log2.log.warn("SignalClient error:", err, {
+        F: __dxlog_file4,
+        L: 135,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
+    this._scheduleReconcileAfterError();
+  }
+  async _close() {
     (0, import_log2.log)("closing...", void 0, {
-      F: __dxlog_file3,
-      L: 173,
+      F: __dxlog_file4,
+      L: 141,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -767,14 +1072,11 @@ var SignalClient = class {
     ].includes(this._state)) {
       return;
     }
-    await this._ctx?.dispose();
-    this._clientReady.reset();
-    await this._client?.close();
-    this._client = void 0;
     this._setState(import_signal.SignalState.CLOSED);
+    await this._safeResetClient();
     (0, import_log2.log)("closed", void 0, {
-      F: __dxlog_file3,
-      L: 184,
+      F: __dxlog_file4,
+      L: 149,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -785,228 +1087,181 @@ var SignalClient = class {
       state: this._state,
       error: this._lastError?.message,
       reconnectIn: this._reconnectAfter,
-      connectionStarted: this._connectionStarted,
-      lastStateChange: this._lastStateChange
+      ...this._monitor.getRecordedTimestamps()
     };
   }
-  async join({ topic, peerId }) {
+  async join(args) {
     (0, import_log2.log)("joining", {
-      topic,
-      peerId
+      topic: args.topic,
+      peerId: args.peerId
     }, {
-      F: __dxlog_file3,
-      L: 199,
+      F: __dxlog_file4,
+      L: 163,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._performance.joinCounter++;
-    this._joinedTopics.add({
-      topic,
-      peerId
-    });
-    this._reconcileTask.schedule();
+    this._monitor.recordJoin();
+    this.localState.join(args);
+    this._reconcileTask?.schedule();
   }
-  async leave({ topic, peerId }) {
-    this._performance.leaveCounter++;
+  async leave(args) {
     (0, import_log2.log)("leaving", {
-      topic,
-      peerId
+      topic: args.topic,
+      peerId: args.peerId
     }, {
-      F: __dxlog_file3,
-      L: 207,
+      F: __dxlog_file4,
+      L: 170,
       S: this,
       C: (f, a) => f(...a)
     });
-    void this._swarmStreams.get({
-      topic,
-      peerId
-    })?.close();
-    this._swarmStreams.delete({
-      topic,
-      peerId
-    });
-    this._joinedTopics.delete({
-      topic,
-      peerId
-    });
-  }
-  async sendMessage(msg) {
-    this._performance.sentMessages++;
-    await this._clientReady.wait();
-    (0, import_invariant2.invariant)(this._state === import_signal.SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 216,
-      S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
+    this._monitor.recordLeave();
+    this.localState.leave(args);
+  }
+  async sendMessage(msg) {
+    return this._monitor.recordMessageSending(msg, async () => {
+      await this._clientReady.wait();
+      (0, import_invariant2.invariant)(this._state === import_signal.SignalState.CONNECTED, "Not connected to Signal Server", {
+        F: __dxlog_file4,
+        L: 178,
+        S: this,
+        A: [
+          "this._state === SignalState.CONNECTED",
+          "'Not connected to Signal Server'"
+        ]
+      });
+      await this._client.sendMessage(msg);
     });
-    await this._client.sendMessage(msg);
   }
   async subscribeMessages(peerId) {
     (0, import_log2.log)("subscribing to messages", {
       peerId
     }, {
-      F: __dxlog_file3,
-      L: 221,
+      F: __dxlog_file4,
+      L: 184,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._subscribedMessages.add({
-      peerId
-    });
-    this._reconcileTask.schedule();
+    this.localState.subscribeMessages(peerId);
+    this._reconcileTask?.schedule();
   }
   async unsubscribeMessages(peerId) {
     (0, import_log2.log)("unsubscribing from messages", {
       peerId
     }, {
-      F: __dxlog_file3,
-      L: 227,
+      F: __dxlog_file4,
+      L: 190,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._subscribedMessages.delete({
-      peerId
-    });
-    void this._messageStreams.get(peerId)?.close();
-    this._messageStreams.delete(peerId);
+    this.localState.unsubscribeMessages(peerId);
   }
   _scheduleReconcileAfterError() {
-    (0, import_async2.scheduleTask)(this._ctx, () => {
-      this._reconcileTask.schedule();
-    }, ERROR_RECONCILE_DELAY);
-  }
-  _setState(newState) {
-    this._state = newState;
-    this._lastStateChange = /* @__PURE__ */ new Date();
-    (0, import_log2.log)("signal state changed", {
-      status: this.getStatus()
-    }, {
-      F: __dxlog_file3,
-      L: 246,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    this.statusChanged.emit(this.getStatus());
+    (0, import_async2.scheduleTask)(this._ctx, () => this._reconcileTask.schedule(), ERROR_RECONCILE_DELAY);
   }
   _createClient() {
     (0, import_log2.log)("creating client", {
       host: this._host,
       state: this._state
     }, {
-      F: __dxlog_file3,
-      L: 251,
+      F: __dxlog_file4,
+      L: 199,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant2.invariant)(!this._client, "Client already created", {
-      F: __dxlog_file3,
-      L: 252,
+      F: __dxlog_file4,
+      L: 200,
       S: this,
       A: [
         "!this._client",
         "'Client already created'"
       ]
     });
-    this._connectionStarted = /* @__PURE__ */ new Date();
+    this._monitor.recordConnectionStartTime();
     this._connectionCtx = this._ctx.derive();
     this._connectionCtx.onDispose(async () => {
       (0, import_log2.log)("connection context disposed", void 0, {
-        F: __dxlog_file3,
-        L: 259,
+        F: __dxlog_file4,
+        L: 207,
         S: this,
         C: (f, a) => f(...a)
       });
-      await Promise.all(Array.from(this._swarmStreams.values()).map((stream) => stream.close()));
-      await Promise.all(Array.from(this._messageStreams.values()).map((stream) => stream.close()));
-      this._swarmStreams.clear();
-      this._messageStreams.clear();
+      const { failureCount } = await this.localState.safeCloseStreams();
+      this._monitor.recordStreamCloseErrors(failureCount);
     });
     try {
-      this._client = new SignalRPCClient({
+      const client = new SignalRPCClient({
         url: this._host,
         callbacks: {
           onConnected: () => {
-            (0, import_log2.log)("socket connected", void 0, {
-              F: __dxlog_file3,
-              L: 271,
-              S: this,
-              C: (f, a) => f(...a)
-            });
-            this._lastError = void 0;
-            this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
-            this._setState(import_signal.SignalState.CONNECTED);
-            this._clientReady.wake();
-            this._reconcileTask.schedule();
+            if (client === this._client) {
+              (0, import_log2.log)("socket connected", void 0, {
+                F: __dxlog_file4,
+                L: 218,
+                S: this,
+                C: (f, a) => f(...a)
+              });
+              this._onConnected();
+            }
           },
           onDisconnected: () => {
+            if (client !== this._client) {
+              return;
+            }
             (0, import_log2.log)("socket disconnected", {
               state: this._state
             }, {
-              F: __dxlog_file3,
-              L: 280,
+              F: __dxlog_file4,
+              L: 227,
               S: this,
               C: (f, a) => f(...a)
             });
             if (this._state === import_signal.SignalState.ERROR) {
               this._setState(import_signal.SignalState.DISCONNECTED);
-              return;
-            }
-            if (this._state !== import_signal.SignalState.CONNECTED && this._state !== import_signal.SignalState.CONNECTING) {
-              this._incrementReconnectTimeout();
+            } else {
+              this._onDisconnected();
             }
-            this._setState(import_signal.SignalState.DISCONNECTED);
-            this._reconnectTask.schedule();
           },
           onError: (error) => {
-            (0, import_log2.log)("socket error", {
-              error,
-              state: this._state
-            }, {
-              F: __dxlog_file3,
-              L: 295,
-              S: this,
-              C: (f, a) => f(...a)
-            });
-            this._lastError = error;
-            if (this._state !== import_signal.SignalState.CONNECTED && this._state !== import_signal.SignalState.CONNECTING) {
-              this._incrementReconnectTimeout();
+            if (client === this._client) {
+              (0, import_log2.log)("socket error", {
+                error,
+                state: this._state
+              }, {
+                F: __dxlog_file4,
+                L: 239,
+                S: this,
+                C: (f, a) => f(...a)
+              });
+              this._onDisconnected({
+                error
+              });
             }
-            this._setState(import_signal.SignalState.ERROR);
-            this._reconnectTask.schedule();
           },
           getMetadata: this._getMetadata
         }
       });
-    } catch (err) {
-      if (this._state !== import_signal.SignalState.CONNECTED && this._state !== import_signal.SignalState.CONNECTING) {
-        this._incrementReconnectTimeout();
-      }
-      this._lastError = err;
-      this._setState(import_signal.SignalState.DISCONNECTED);
-      this._reconnectTask.schedule();
+      this._client = client;
+    } catch (error) {
+      this._client = void 0;
+      this._onDisconnected({
+        error
+      });
     }
   }
-  _incrementReconnectTimeout() {
-    this._reconnectAfter *= 2;
-    this._reconnectAfter = Math.min(this._reconnectAfter, MAX_RECONNECT_TIMEOUT);
-  }
   async _reconnect() {
     (0, import_log2.log)(`reconnecting in ${this._reconnectAfter}ms`, {
       state: this._state
     }, {
-      F: __dxlog_file3,
-      L: 323,
+      F: __dxlog_file4,
+      L: 254,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._performance.reconnectCounter++;
     if (this._state === import_signal.SignalState.RECONNECTING) {
-      import_log2.log.warn("Signal api already reconnecting.", void 0, {
-        F: __dxlog_file3,
-        L: 327,
+      import_log2.log.info("Signal api already reconnecting.", void 0, {
+        F: __dxlog_file4,
+        L: 257,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1015,137 +1270,90 @@ var SignalClient = class {
     if (this._state === import_signal.SignalState.CLOSED) {
       return;
     }
-    this._clientReady.reset();
-    await this._connectionCtx?.dispose();
-    this._client?.close().catch(() => {
-    });
-    this._client = void 0;
-    await (0, import_context2.cancelWithContext)(this._ctx, (0, import_async2.sleep)(this._reconnectAfter));
     this._setState(import_signal.SignalState.RECONNECTING);
+    await this._safeResetClient();
+    await (0, import_context2.cancelWithContext)(this._ctx, (0, import_async2.sleep)(this._reconnectAfter));
     this._createClient();
   }
-  async _reconcileSwarmSubscriptions() {
-    await (0, import_async2.asyncTimeout)((0, import_context2.cancelWithContext)(this._connectionCtx, this._clientReady.wait()), 5e3);
-    const client = this._client;
-    (0, import_invariant2.invariant)(this._state === import_signal.SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 352,
-      S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
-    });
-    for (const { topic, peerId } of this._swarmStreams.keys()) {
-      if (this._joinedTopics.has({
-        topic,
-        peerId
-      })) {
-        continue;
-      }
-      void this._swarmStreams.get({
-        topic,
-        peerId
-      })?.close();
-      this._swarmStreams.delete({
-        topic,
-        peerId
-      });
+  _onConnected() {
+    this._lastError = void 0;
+    this._lastReconciliationFailed = false;
+    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
+    this._setState(import_signal.SignalState.CONNECTED);
+    this._clientReady.wake();
+    this._reconcileTask.schedule();
+  }
+  _onDisconnected(options) {
+    this._updateReconnectTimeout();
+    if (this._state === import_signal.SignalState.CLOSED) {
+      return;
     }
-    for (const { topic, peerId } of this._joinedTopics.values()) {
-      if (this._swarmStreams.has({
-        topic,
-        peerId
-      })) {
-        continue;
-      }
-      const swarmStream = await (0, import_async2.asyncTimeout)((0, import_context2.cancelWithContext)(this._connectionCtx, client.join({
-        topic,
-        peerId
-      })), 5e3);
-      swarmStream.subscribe(async (swarmEvent) => {
-        (0, import_log2.log)("swarm event", {
-          swarmEvent
-        }, {
-          F: __dxlog_file3,
-          L: 379,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        await this._onSwarmEvent({
-          topic,
-          swarmEvent
-        });
-      });
-      this._swarmStreams.set({
-        topic,
-        peerId
-      }, swarmStream);
+    if (options?.error) {
+      this._lastError = options.error;
+      this._setState(import_signal.SignalState.ERROR);
+    } else {
+      this._setState(import_signal.SignalState.DISCONNECTED);
     }
+    this._reconnectTask.schedule();
   }
-  async _reconcileMessageSubscriptions() {
-    await (0, import_async2.asyncTimeout)((0, import_context2.cancelWithContext)(this._connectionCtx, this._clientReady.wait()), 5e3);
-    const client = this._client;
-    (0, import_invariant2.invariant)(this._state === import_signal.SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 392,
+  _setState(newState) {
+    this._state = newState;
+    this._monitor.recordStateChangeTime();
+    (0, import_log2.log)("signal state changed", {
+      status: this.getStatus()
+    }, {
+      F: __dxlog_file4,
+      L: 298,
       S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
+      C: (f, a) => f(...a)
     });
-    for (const peerId of this._messageStreams.keys()) {
-      if (this._subscribedMessages.has({
-        peerId
-      })) {
-        continue;
-      }
-      void this._messageStreams.get(peerId)?.close();
-      this._messageStreams.delete(peerId);
-    }
-    for (const { peerId } of this._subscribedMessages.values()) {
-      if (this._messageStreams.has(peerId)) {
-        continue;
-      }
-      const messageStream = await (0, import_async2.asyncTimeout)((0, import_context2.cancelWithContext)(this._connectionCtx, client.receiveMessages(peerId)), 5e3);
-      messageStream.subscribe(async (message) => {
-        this._performance.receivedMessages++;
-        await this._onMessage({
-          author: import_keys2.PublicKey.from(message.author),
-          recipient: import_keys2.PublicKey.from(message.recipient),
-          payload: message.payload
-        });
-      });
-      this._messageStreams.set(peerId, messageStream);
+    this.statusChanged.emit(this.getStatus());
+  }
+  _updateReconnectTimeout() {
+    if (this._state !== import_signal.SignalState.CONNECTED && this._state !== import_signal.SignalState.CONNECTING) {
+      this._reconnectAfter *= 2;
+      this._reconnectAfter = Math.min(this._reconnectAfter, MAX_RECONNECT_TIMEOUT);
     }
   }
+  async _safeResetClient() {
+    await this._connectionCtx?.dispose();
+    this._connectionCtx = void 0;
+    this._clientReady.reset();
+    await this._client?.close().catch(() => {
+    });
+    this._client = void 0;
+  }
 };
-var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
+var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
 var MemorySignalManagerContext = class {
   constructor() {
-    this.swarmEvent = new import_async4.Event();
-    this.swarms = new import_util3.ComplexMap(import_keys4.PublicKey.hash);
-    this.connections = new import_util3.ComplexMap(import_keys4.PublicKey.hash);
+    this.swarmEvent = new import_async5.Event();
+    this.swarms = new import_util3.ComplexMap(import_keys5.PublicKey.hash);
+    this.connections = new import_util3.ComplexMap(import_keys5.PublicKey.hash);
   }
 };
 var MemorySignalManager = class {
   constructor(_context) {
     this._context = _context;
-    this.statusChanged = new import_async4.Event();
-    this.commandTrace = new import_async4.Event();
-    this.swarmEvent = new import_async4.Event();
-    this.onMessage = new import_async4.Event();
+    this.statusChanged = new import_async5.Event();
+    this.swarmEvent = new import_async5.Event();
+    this.onMessage = new import_async5.Event();
     this._joinedSwarms = new import_util3.ComplexSet(({ topic, peerId }) => topic.toHex() + peerId.toHex());
-    this._freezeTrigger = new import_async4.Trigger().wake();
-    this._ctx = new import_context4.Context();
+    this._freezeTrigger = new import_async5.Trigger().wake();
+    this._ctx = new import_context5.Context(void 0, {
+      F: __dxlog_file5,
+      L: 62
+    });
     this._ctx.onDispose(this._context.swarmEvent.on((data) => this.swarmEvent.emit(data)));
   }
   async open() {
     if (!this._ctx.disposed) {
       return;
     }
-    this._ctx = new import_context4.Context();
+    this._ctx = new import_context5.Context(void 0, {
+      F: __dxlog_file5,
+      L: 71
+    });
     this._ctx.onDispose(this._context.swarmEvent.on((data) => this.swarmEvent.emit(data)));
     await Promise.all([
       ...this._joinedSwarms.values()
@@ -1169,8 +1377,8 @@ var MemorySignalManager = class {
   }
   async join({ topic, peerId }) {
     (0, import_invariant4.invariant)(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 102,
+      F: __dxlog_file5,
+      L: 100,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1182,7 +1390,7 @@ var MemorySignalManager = class {
       peerId
     });
     if (!this._context.swarms.has(topic)) {
-      this._context.swarms.set(topic, new import_util3.ComplexSet(import_keys4.PublicKey.hash));
+      this._context.swarms.set(topic, new import_util3.ComplexSet(import_keys5.PublicKey.hash));
     }
     this._context.swarms.get(topic).add(peerId);
     this._context.swarmEvent.emit({
@@ -1210,8 +1418,8 @@ var MemorySignalManager = class {
   }
   async leave({ topic, peerId }) {
     (0, import_invariant4.invariant)(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 138,
+      F: __dxlog_file5,
+      L: 136,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1223,7 +1431,7 @@ var MemorySignalManager = class {
       peerId
     });
     if (!this._context.swarms.has(topic)) {
-      this._context.swarms.set(topic, new import_util3.ComplexSet(import_keys4.PublicKey.hash));
+      this._context.swarms.set(topic, new import_util3.ComplexSet(import_keys5.PublicKey.hash));
     }
     this._context.swarms.get(topic).delete(peerId);
     const swarmEvent = {
@@ -1237,19 +1445,19 @@ var MemorySignalManager = class {
     });
   }
   async sendMessage({ author, recipient, payload }) {
-    (0, import_log4.log)("send message", {
+    (0, import_log5.log)("send message", {
       author,
       recipient,
       ...dec(payload)
     }, {
-      F: __dxlog_file4,
-      L: 158,
+      F: __dxlog_file5,
+      L: 156,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant4.invariant)(recipient, void 0, {
-      F: __dxlog_file4,
-      L: 160,
+      F: __dxlog_file5,
+      L: 158,
       S: this,
       A: [
         "recipient",
@@ -1257,8 +1465,8 @@ var MemorySignalManager = class {
       ]
     });
     (0, import_invariant4.invariant)(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 161,
+      F: __dxlog_file5,
+      L: 159,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1268,24 +1476,24 @@ var MemorySignalManager = class {
     await this._freezeTrigger.wait();
     const remote = this._context.connections.get(recipient);
     if (!remote) {
-      import_log4.log.warn("recipient is not subscribed for messages", {
+      import_log5.log.warn("recipient is not subscribed for messages", {
         author,
         recipient
       }, {
-        F: __dxlog_file4,
-        L: 167,
+        F: __dxlog_file5,
+        L: 165,
         S: this,
         C: (f, a) => f(...a)
       });
       return;
     }
     if (remote._ctx.disposed) {
-      import_log4.log.warn("recipient is disposed", {
+      import_log5.log.warn("recipient is disposed", {
         author,
         recipient
       }, {
-        F: __dxlog_file4,
-        L: 172,
+        F: __dxlog_file5,
+        L: 170,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1293,24 +1501,24 @@ var MemorySignalManager = class {
     }
     remote._freezeTrigger.wait().then(() => {
       if (remote._ctx.disposed) {
-        import_log4.log.warn("recipient is disposed", {
+        import_log5.log.warn("recipient is disposed", {
           author,
           recipient
         }, {
-          F: __dxlog_file4,
-          L: 180,
+          F: __dxlog_file5,
+          L: 178,
           S: this,
           C: (f, a) => f(...a)
         });
         return;
       }
-      (0, import_log4.log)("receive message", {
+      (0, import_log5.log)("receive message", {
         author,
         recipient,
         ...dec(payload)
       }, {
-        F: __dxlog_file4,
-        L: 184,
+        F: __dxlog_file5,
+        L: 182,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1320,33 +1528,33 @@ var MemorySignalManager = class {
         payload
       });
     }).catch((err) => {
-      import_log4.log.error("error while waiting for freeze", {
+      import_log5.log.error("error while waiting for freeze", {
         err
       }, {
-        F: __dxlog_file4,
-        L: 189,
+        F: __dxlog_file5,
+        L: 187,
         S: this,
         C: (f, a) => f(...a)
       });
     });
   }
   async subscribeMessages(peerId) {
-    (0, import_log4.log)("subscribing", {
+    (0, import_log5.log)("subscribing", {
       peerId
     }, {
-      F: __dxlog_file4,
-      L: 194,
+      F: __dxlog_file5,
+      L: 192,
       S: this,
       C: (f, a) => f(...a)
     });
     this._context.connections.set(peerId, this);
   }
   async unsubscribeMessages(peerId) {
-    (0, import_log4.log)("unsubscribing", {
+    (0, import_log5.log)("unsubscribing", {
       peerId
     }, {
-      F: __dxlog_file4,
-      L: 199,
+      F: __dxlog_file5,
+      L: 197,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1372,6 +1580,19 @@ var dec = (payload) => {
   }
   return {};
 };
+var WebsocketSignalManagerMonitor = class {
+  recordRateLimitExceeded() {
+    import_tracing4.trace.metrics.increment("mesh.signal.signal-manager.rate-limit-hit", 1);
+  }
+  recordServerFailure(params) {
+    import_tracing4.trace.metrics.increment("mesh.signal.signal-manager.server-failure", 1, {
+      tags: {
+        server: params.serverName,
+        restarted: params.willRestart
+      }
+    });
+  }
+};
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1382,7 +1603,7 @@ function _ts_decorate(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
+var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
 var MAX_SERVER_FAILURES = 5;
 var WSS_SIGNAL_SERVER_REBOOT_DELAY = 3e3;
 var WebsocketSignalManager = class {
@@ -1390,18 +1611,18 @@ var WebsocketSignalManager = class {
     this._hosts = _hosts;
     this._getMetadata = _getMetadata;
     this._servers = /* @__PURE__ */ new Map();
+    this._monitor = new WebsocketSignalManagerMonitor();
     this._opened = false;
     this.failureCount = /* @__PURE__ */ new Map();
-    this.statusChanged = new import_async5.Event();
-    this.commandTrace = new import_async5.Event();
-    this.swarmEvent = new import_async5.Event();
-    this.onMessage = new import_async5.Event();
-    this._instanceId = import_keys5.PublicKey.random().toHex();
-    (0, import_log5.log)("Created WebsocketSignalManager", {
+    this.statusChanged = new import_async6.Event();
+    this.swarmEvent = new import_async6.Event();
+    this.onMessage = new import_async6.Event();
+    this._instanceId = import_keys6.PublicKey.random().toHex();
+    (0, import_log6.log)("Created WebsocketSignalManager", {
       hosts: this._hosts
     }, {
-      F: __dxlog_file5,
-      L: 51,
+      F: __dxlog_file6,
+      L: 58,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1413,39 +1634,37 @@ var WebsocketSignalManager = class {
       server.statusChanged.on(() => this.statusChanged.emit(this.getStatus()));
       this._servers.set(host.server, server);
       this.failureCount.set(host.server, 0);
-      server.commandTrace.on((trace5) => this.commandTrace.emit(trace5));
     }
+    this._failedServersBitfield = import_util4.BitField.zeros(this._hosts.length);
   }
   async open() {
     if (this._opened) {
       return;
     }
-    (0, import_log5.log)("open signal manager", {
+    (0, import_log6.log)("open signal manager", {
       hosts: this._hosts
     }, {
-      F: __dxlog_file5,
-      L: 78,
+      F: __dxlog_file6,
+      L: 85,
       S: this,
       C: (f, a) => f(...a)
     });
-    import_log5.log.trace("dxos.mesh.websocket-signal-manager.open", import_protocols5.trace.begin({
+    import_log6.log.trace("dxos.mesh.websocket-signal-manager.open", import_protocols5.trace.begin({
       id: this._instanceId
     }), {
-      F: __dxlog_file5,
-      L: 79,
+      F: __dxlog_file6,
+      L: 86,
       S: this,
       C: (f, a) => f(...a)
     });
     this._initContext();
-    [
-      ...this._servers.values()
-    ].forEach((server) => server.open());
+    await (0, import_util4.safeAwaitAll)(this._servers.values(), (server) => server.open());
     this._opened = true;
-    import_log5.log.trace("dxos.mesh.websocket-signal-manager.open", import_protocols5.trace.end({
+    import_log6.log.trace("dxos.mesh.websocket-signal-manager.open", import_protocols5.trace.end({
       id: this._instanceId
     }), {
-      F: __dxlog_file5,
-      L: 87,
+      F: __dxlog_file6,
+      L: 93,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1456,20 +1675,20 @@ var WebsocketSignalManager = class {
     }
     this._opened = false;
     await this._ctx.dispose();
-    await Promise.all(Array.from(this._servers.values()).map((server) => server.close()));
+    await (0, import_util4.safeAwaitAll)(this._servers.values(), (server) => server.close());
   }
   async restartServer(serverName) {
-    (0, import_log5.log)("restarting server", {
+    (0, import_log6.log)("restarting server", {
       serverName
     }, {
-      F: __dxlog_file5,
-      L: 103,
+      F: __dxlog_file6,
+      L: 107,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "server already closed", {
-      F: __dxlog_file5,
-      L: 104,
+      F: __dxlog_file6,
+      L: 108,
       S: this,
       A: [
         "this._opened",
@@ -1478,8 +1697,8 @@ var WebsocketSignalManager = class {
     });
     const server = this._servers.get(serverName);
     (0, import_invariant5.invariant)(server, "server not found", {
-      F: __dxlog_file5,
-      L: 107,
+      F: __dxlog_file6,
+      L: 111,
       S: this,
       A: [
         "server",
@@ -1487,25 +1706,25 @@ var WebsocketSignalManager = class {
       ]
     });
     await server.close();
-    await (0, import_async5.sleep)(WSS_SIGNAL_SERVER_REBOOT_DELAY);
+    await (0, import_async6.sleep)(WSS_SIGNAL_SERVER_REBOOT_DELAY);
     await server.open();
   }
   getStatus() {
     return Array.from(this._servers.values()).map((server) => server.getStatus());
   }
   async join({ topic, peerId }) {
-    (0, import_log5.log)("join", {
+    (0, import_log6.log)("join", {
       topic,
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 120,
+      F: __dxlog_file6,
+      L: 124,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 121,
+      F: __dxlog_file6,
+      L: 125,
       S: this,
       A: [
         "this._opened",
@@ -1518,18 +1737,18 @@ var WebsocketSignalManager = class {
     }));
   }
   async leave({ topic, peerId }) {
-    (0, import_log5.log)("leaving", {
+    (0, import_log6.log)("leaving", {
       topic,
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 127,
+      F: __dxlog_file6,
+      L: 131,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 128,
+      F: __dxlog_file6,
+      L: 132,
       S: this,
       A: [
         "this._opened",
@@ -1542,89 +1761,115 @@ var WebsocketSignalManager = class {
     }));
   }
   async sendMessage({ author, recipient, payload }) {
-    (0, import_log5.log)("signal", {
+    (0, import_log6.log)("signal", {
       recipient
     }, {
-      F: __dxlog_file5,
-      L: 142,
+      F: __dxlog_file6,
+      L: 145,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 143,
+      F: __dxlog_file6,
+      L: 146,
       S: this,
       A: [
         "this._opened",
         "'Closed'"
       ]
     });
-    void this._forEachServer(async (server, serverName) => {
+    void this._forEachServer(async (server, serverName, index) => {
       void server.sendMessage({
         author,
         recipient,
         payload
-      }).catch((err) => {
+      }).then(() => this._clearServerFailedFlag(serverName, index)).catch((err) => {
         if (err instanceof import_protocols5.RateLimitExceededError) {
-          import_log5.log.info("WSS rate limit exceeded", {
+          import_log6.log.info("WSS rate limit exceeded", {
             err
           }, {
-            F: __dxlog_file5,
-            L: 148,
+            F: __dxlog_file6,
+            L: 154,
             S: this,
             C: (f, a) => f(...a)
           });
+          this._monitor.recordRateLimitExceeded();
         } else if (err instanceof import_protocols5.TimeoutError || err.constructor.name === "TimeoutError") {
-          import_log5.log.info("WSS sendMessage timeout", {
+          import_log6.log.info("WSS sendMessage timeout", {
             err
           }, {
-            F: __dxlog_file5,
-            L: 150,
+            F: __dxlog_file6,
+            L: 157,
             S: this,
             C: (f, a) => f(...a)
           });
-          void this.checkServerFailure(serverName);
+          void this.checkServerFailure(serverName, index);
         } else {
-          import_log5.log.info(`error sending to ${serverName}`, {
+          import_log6.log.warn(`error sending to ${serverName}`, {
             err
           }, {
-            F: __dxlog_file5,
-            L: 153,
+            F: __dxlog_file6,
+            L: 160,
             S: this,
             C: (f, a) => f(...a)
           });
-          void this.checkServerFailure(serverName);
+          void this.checkServerFailure(serverName, index);
         }
       });
     });
   }
-  async checkServerFailure(serverName) {
+  async checkServerFailure(serverName, index) {
     const failureCount = this.failureCount.get(serverName) ?? 0;
-    if (failureCount > MAX_SERVER_FAILURES) {
-      import_log5.log.warn(`too many failures sending to ${serverName} (${failureCount} > ${MAX_SERVER_FAILURES}), restarting`, void 0, {
-        F: __dxlog_file5,
-        L: 164,
-        S: this,
-        C: (f, a) => f(...a)
-      });
+    const isRestartRequired = failureCount > MAX_SERVER_FAILURES;
+    this._monitor.recordServerFailure({
+      serverName,
+      willRestart: isRestartRequired
+    });
+    if (isRestartRequired) {
+      if (!import_util4.BitField.get(this._failedServersBitfield, index)) {
+        import_log6.log.warn("too many failures for ws-server, restarting", {
+          serverName,
+          failureCount
+        }, {
+          F: __dxlog_file6,
+          L: 174,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        import_util4.BitField.set(this._failedServersBitfield, index, true);
+      }
       await this.restartServer(serverName);
       this.failureCount.set(serverName, 0);
       return;
     }
     this.failureCount.set(serverName, (this.failureCount.get(serverName) ?? 0) + 1);
   }
+  _clearServerFailedFlag(serverName, index) {
+    if (import_util4.BitField.get(this._failedServersBitfield, index)) {
+      import_log6.log.info("server connection restored", {
+        serverName
+      }, {
+        F: __dxlog_file6,
+        L: 187,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      import_util4.BitField.set(this._failedServersBitfield, index, false);
+      this.failureCount.set(serverName, 0);
+    }
+  }
   async subscribeMessages(peerId) {
-    (0, import_log5.log)("subscribed for message stream", {
+    (0, import_log6.log)("subscribed for message stream", {
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 174,
+      F: __dxlog_file6,
+      L: 194,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 175,
+      F: __dxlog_file6,
+      L: 195,
       S: this,
       A: [
         "this._opened",
@@ -1634,17 +1879,17 @@ var WebsocketSignalManager = class {
     await this._forEachServer(async (server) => server.subscribeMessages(peerId));
   }
   async unsubscribeMessages(peerId) {
-    (0, import_log5.log)("subscribed for message stream", {
+    (0, import_log6.log)("subscribed for message stream", {
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 181,
+      F: __dxlog_file6,
+      L: 201,
       S: this,
       C: (f, a) => f(...a)
     });
     (0, import_invariant5.invariant)(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 182,
+      F: __dxlog_file6,
+      L: 202,
       S: this,
       A: [
         "this._opened",
@@ -1654,42 +1899,45 @@ var WebsocketSignalManager = class {
     await this._forEachServer(async (server) => server.unsubscribeMessages(peerId));
   }
   _initContext() {
-    this._ctx = new import_context5.Context({
-      onError: (err) => import_log5.log.catch(err, void 0, {
-        F: __dxlog_file5,
-        L: 189,
+    this._ctx = new import_context6.Context({
+      onError: (err) => import_log6.log.catch(err, void 0, {
+        F: __dxlog_file6,
+        L: 209,
         S: this,
         C: (f, a) => f(...a)
       })
+    }, {
+      F: __dxlog_file6,
+      L: 208
     });
   }
   async _forEachServer(fn) {
-    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server]) => fn(server, serverName)));
+    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server], idx) => fn(server, serverName, idx)));
   }
 };
 _ts_decorate([
-  import_async5.synchronized
+  import_async6.synchronized
 ], WebsocketSignalManager.prototype, "open", null);
 _ts_decorate([
-  import_async5.synchronized
+  import_async6.synchronized
 ], WebsocketSignalManager.prototype, "close", null);
 _ts_decorate([
-  import_async5.synchronized
+  import_async6.synchronized
 ], WebsocketSignalManager.prototype, "join", null);
 _ts_decorate([
-  import_async5.synchronized
+  import_async6.synchronized
 ], WebsocketSignalManager.prototype, "leave", null);
 _ts_decorate([
-  import_async5.synchronized
+  import_async6.synchronized
 ], WebsocketSignalManager.prototype, "checkServerFailure", null);
-var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/utils.ts";
+var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/utils.ts";
 var setIdentityTags = ({ identityService, devicesService, setTag }) => {
   identityService.queryIdentity().subscribe((idqr) => {
     if (!idqr?.identity?.identityKey) {
-      (0, import_log6.log)("empty response from identity service", {
+      (0, import_log7.log)("empty response from identity service", {
         idqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 21,
         S: void 0,
         C: (f, a) => f(...a)
@@ -1700,10 +1948,10 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
   });
   devicesService.queryDevices().subscribe((dqr) => {
     if (!dqr || !dqr.devices || dqr.devices.length === 0) {
-      (0, import_log6.log)("empty response from device service", {
+      (0, import_log7.log)("empty response from device service", {
         device: dqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 30,
         S: void 0,
         C: (f, a) => f(...a)
@@ -1711,7 +1959,7 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
       return;
     }
     (0, import_invariant6.invariant)(dqr, "empty response from device service", {
-      F: __dxlog_file6,
+      F: __dxlog_file7,
       L: 33,
       S: void 0,
       A: [
@@ -1721,10 +1969,10 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
     });
     const thisDevice = dqr.devices.find((device) => device.kind === import_services.DeviceKind.CURRENT);
     if (!thisDevice) {
-      (0, import_log6.log)("no current device", {
+      (0, import_log7.log)("no current device", {
         device: dqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 37,
         S: void 0,
         C: (f, a) => f(...a)