@dxos/messaging 0.5.9-main.07b4bad → 0.5.9-main.1c1903d

package/dist/lib/browser/index.mjs

@@ -13,9 +13,25 @@ import { Context } from "@dxos/context";
 import { invariant } from "@dxos/invariant";
 import { PublicKey } from "@dxos/keys";
 import { log } from "@dxos/log";
-import { TimeoutError as ProtocolTimeoutError, schema, trace } from "@dxos/protocols";
+import { TimeoutError as ProtocolTimeoutError, schema, trace as trace2 } from "@dxos/protocols";
 import { ComplexMap, ComplexSet } from "@dxos/util";
 
+// packages/core/mesh/messaging/src/messenger-monitor.ts
+import { trace } from "@dxos/tracing";
+var MessengerMonitor = class {
+  recordMessageAckFailed() {
+    trace.metrics.increment("mesh.signal.messenger.failed-ack", 1);
+  }
+  recordReliableMessage(params) {
+    trace.metrics.increment("mesh.signal.messenger.reliable-send", 1, {
+      tags: {
+        success: params.sent,
+        attempts: params.sendAttempts
+      }
+    });
+  }
+};
+
 // packages/core/mesh/messaging/src/timeouts.ts
 var MESSAGE_TIMEOUT = 1e4;
 
@@ -26,6 +42,7 @@ var Acknowledgement = schema.getCodecForType("dxos.mesh.messaging.Acknowledgemen
 var RECEIVED_MESSAGES_GC_INTERVAL = 12e4;
 var Messenger = class {
   constructor({ signalManager, retryDelay = 300 }) {
+    this._monitor = new MessengerMonitor();
     // { peerId, payloadType } => listeners set
     this._listeners = new ComplexMap(({ peerId, payloadType }) => peerId.toHex() + payloadType);
     // peerId => listeners set
@@ -46,18 +63,18 @@ var Messenger = class {
       return;
     }
     const traceId = PublicKey.random().toHex();
-    log.trace("dxos.mesh.messenger.open", trace.begin({
+    log.trace("dxos.mesh.messenger.open", trace2.begin({
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 69,
+      L: 71,
       S: this,
       C: (f, a) => f(...a)
     });
     this._ctx = new Context({
       onError: (err) => log.catch(err, void 0, {
         F: __dxlog_file,
-        L: 71,
+        L: 73,
         S: this,
         C: (f, a) => f(...a)
       })
@@ -67,7 +84,7 @@ var Messenger = class {
         from: message.author
       }, {
         F: __dxlog_file,
-        L: 75,
+        L: 77,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -77,11 +94,11 @@ var Messenger = class {
       this._performGc();
     }, RECEIVED_MESSAGES_GC_INTERVAL);
     this._closed = false;
-    log.trace("dxos.mesh.messenger.open", trace.end({
+    log.trace("dxos.mesh.messenger.open", trace2.end({
       id: traceId
     }), {
       F: __dxlog_file,
-      L: 90,
+      L: 92,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -96,7 +113,7 @@ var Messenger = class {
   async sendMessage({ author, recipient, payload }) {
     invariant(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 102,
+      L: 104,
       S: this,
       A: [
         "!this._closed",
@@ -110,7 +127,7 @@ var Messenger = class {
     };
     invariant(!this._onAckCallbacks.has(reliablePayload.messageId), void 0, {
       F: __dxlog_file,
-      L: 109,
+      L: 111,
       S: this,
       A: [
         "!this._onAckCallbacks.has(reliablePayload.messageId!)",
@@ -123,12 +140,13 @@ var Messenger = class {
       recipient
     }, {
       F: __dxlog_file,
-      L: 110,
+      L: 112,
       S: this,
       C: (f, a) => f(...a)
     });
     let messageReceived;
     let timeoutHit;
+    let sendAttempts = 0;
     const promise = new Promise((resolve, reject) => {
       messageReceived = resolve;
       timeoutHit = reject;
@@ -138,10 +156,11 @@ var Messenger = class {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 124,
+        L: 127,
         S: this,
         C: (f, a) => f(...a)
       });
+      sendAttempts++;
       await this._encodeAndSend({
         author,
         recipient,
@@ -150,7 +169,7 @@ var Messenger = class {
         err
       }, {
         F: __dxlog_file,
-        L: 126,
+        L: 130,
         S: this,
         C: (f, a) => f(...a)
       }));
@@ -160,18 +179,26 @@ var Messenger = class {
         messageId: reliablePayload.messageId
       }, {
         F: __dxlog_file,
-        L: 135,
+        L: 139,
         S: this,
         C: (f, a) => f(...a)
       });
       this._onAckCallbacks.delete(reliablePayload.messageId);
       timeoutHit(new ProtocolTimeoutError("signaling message not delivered", new TimeoutError(MESSAGE_TIMEOUT, "Message not delivered")));
       void messageContext.dispose();
+      this._monitor.recordReliableMessage({
+        sendAttempts,
+        sent: false
+      });
     }, MESSAGE_TIMEOUT);
     this._onAckCallbacks.set(reliablePayload.messageId, () => {
       messageReceived();
       this._onAckCallbacks.delete(reliablePayload.messageId);
       void messageContext.dispose();
+      this._monitor.recordReliableMessage({
+        sendAttempts,
+        sent: true
+      });
     });
     await this._encodeAndSend({
       author,
@@ -187,7 +214,7 @@ var Messenger = class {
   async listen({ peerId, payloadType, onMessage }) {
     invariant(!this._closed, "Closed", {
       F: __dxlog_file,
-      L: 171,
+      L: 177,
       S: this,
       A: [
         "!this._closed",
@@ -251,7 +278,7 @@ var Messenger = class {
   async _handleReliablePayload({ author, recipient, payload }) {
     invariant(payload.type_url === "dxos.mesh.messaging.ReliablePayload", void 0, {
       F: __dxlog_file,
-      L: 232,
+      L: 238,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.ReliablePayload'",
@@ -265,15 +292,20 @@ var Messenger = class {
       messageId: reliablePayload.messageId
     }, {
       F: __dxlog_file,
-      L: 235,
+      L: 241,
       S: this,
       C: (f, a) => f(...a)
     });
-    await this._sendAcknowledgement({
-      author,
-      recipient,
-      messageId: reliablePayload.messageId
-    });
+    try {
+      await this._sendAcknowledgement({
+        author,
+        recipient,
+        messageId: reliablePayload.messageId
+      });
+    } catch (err) {
+      this._monitor.recordMessageAckFailed();
+      throw err;
+    }
     if (this._receivedMessages.has(reliablePayload.messageId)) {
       return;
     }
@@ -287,7 +319,7 @@ var Messenger = class {
   async _handleAcknowledgement({ payload }) {
     invariant(payload.type_url === "dxos.mesh.messaging.Acknowledgement", void 0, {
       F: __dxlog_file,
-      L: 258,
+      L: 269,
       S: this,
       A: [
         "payload.type_url === 'dxos.mesh.messaging.Acknowledgement'",
@@ -303,7 +335,7 @@ var Messenger = class {
       to: author
     }, {
       F: __dxlog_file,
-      L: 271,
+      L: 282,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -354,7 +386,7 @@ var Messenger = class {
         elapsed
       }, {
         F: __dxlog_file,
-        L: 319,
+        L: 330,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -363,37 +395,310 @@ var Messenger = class {
 };
 
 // packages/core/mesh/messaging/src/signal-client/signal-client.ts
-import { DeferredTask, Event, Trigger as Trigger2, asyncTimeout, scheduleTask as scheduleTask2, scheduleTaskInterval as scheduleTaskInterval3, sleep } from "@dxos/async";
-import { Context as Context3, cancelWithContext } from "@dxos/context";
+import { DeferredTask, Event as Event2, Trigger as Trigger2, scheduleTask as scheduleTask2, scheduleTaskInterval as scheduleTaskInterval3, sleep } from "@dxos/async";
+import { cancelWithContext as cancelWithContext2, Resource } from "@dxos/context";
 import { invariant as invariant3 } from "@dxos/invariant";
-import { PublicKey as PublicKey3 } from "@dxos/keys";
-import { log as log3 } from "@dxos/log";
-import { trace as trace3 } from "@dxos/protocols";
+import { PublicKey as PublicKey4 } from "@dxos/keys";
+import { log as log4 } from "@dxos/log";
+import { trace as trace6 } from "@dxos/protocols";
 import { SignalState } from "@dxos/protocols/proto/dxos/mesh/signal";
-import { ComplexMap as ComplexMap2, ComplexSet as ComplexSet2 } from "@dxos/util";
+
+// packages/core/mesh/messaging/src/signal-client/signal-client-monitor.ts
+import { trace as trace3 } from "@dxos/tracing";
+var SignalClientMonitor = class {
+  constructor() {
+    this._performance = {
+      sentMessages: 0,
+      receivedMessages: 0,
+      reconnectCounter: 0,
+      joinCounter: 0,
+      leaveCounter: 0
+    };
+    /**
+    * Timestamp of when the connection attempt was began.
+    */
+    this._connectionStarted = /* @__PURE__ */ new Date();
+    /**
+    * Timestamp of last state change.
+    */
+    this._lastStateChange = /* @__PURE__ */ new Date();
+  }
+  getRecordedTimestamps() {
+    return {
+      connectionStarted: this._connectionStarted,
+      lastStateChange: this._lastStateChange
+    };
+  }
+  recordStateChangeTime() {
+    this._lastStateChange = /* @__PURE__ */ new Date();
+  }
+  recordConnectionStartTime() {
+    this._connectionStarted = /* @__PURE__ */ new Date();
+  }
+  recordReconnect(params) {
+    this._performance.reconnectCounter++;
+    trace3.metrics.increment("mesh.signal.signal-client.reconnect", 1, {
+      tags: {
+        success: params.success
+      }
+    });
+  }
+  recordJoin() {
+    this._performance.joinCounter++;
+  }
+  recordLeave() {
+    this._performance.leaveCounter++;
+  }
+  recordMessageReceived(message) {
+    this._performance.receivedMessages++;
+    trace3.metrics.increment("mesh.signal.signal-client.received-total", 1, {
+      tags: createIdentityTags(message)
+    });
+    trace3.metrics.distribution("mesh.signal.signal-client.bytes-in", getByteCount(message), {
+      tags: createIdentityTags(message)
+    });
+  }
+  async recordMessageSending(message, sendMessage) {
+    this._performance.sentMessages++;
+    const tags = createIdentityTags(message);
+    let success = true;
+    try {
+      const reqStart = Date.now();
+      await sendMessage();
+      const reqDuration = Date.now() - reqStart;
+      trace3.metrics.distribution("mesh.signal.signal-client.send-duration", reqDuration, {
+        tags
+      });
+      trace3.metrics.distribution("mesh.signal.signal-client.bytes-out", getByteCount(message), {
+        tags
+      });
+    } catch (err) {
+      success = false;
+    }
+    trace3.metrics.increment("mesh.signal.signal-client.sent-total", 1, {
+      tags: {
+        ...tags,
+        success
+      }
+    });
+  }
+  recordStreamCloseErrors(count) {
+    trace3.metrics.increment("mesh.signal.signal-client.stream-close-errors", count);
+  }
+  recordReconciliation(params) {
+    trace3.metrics.increment("mesh.signal.signal-client.reconciliation", 1, {
+      tags: {
+        success: params.success
+      }
+    });
+  }
+};
+var getByteCount = (message) => {
+  return message.author.asUint8Array().length + message.recipient.asUint8Array().length + message.payload.type_url.length + message.payload.value.length;
+};
+var createIdentityTags = (message) => {
+  return {
+    peer: message.author.toHex()
+  };
+};
+
+// packages/core/mesh/messaging/src/signal-client/signal-local-state.ts
+import { asyncTimeout, Event } from "@dxos/async";
+import { cancelWithContext } from "@dxos/context";
+import { PublicKey as PublicKey2 } from "@dxos/keys";
+import { log as log2 } from "@dxos/log";
+import { ComplexMap as ComplexMap2, ComplexSet as ComplexSet2, safeAwaitAll } from "@dxos/util";
+var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-local-state.ts";
+var SignalLocalState = class {
+  constructor(_onMessage, _onSwarmEvent) {
+    this._onMessage = _onMessage;
+    this._onSwarmEvent = _onSwarmEvent;
+    this._swarmStreams = new ComplexMap2(({ topic, peerId }) => topic.toHex() + peerId.toHex());
+    this._joinedTopics = new ComplexSet2(({ topic, peerId }) => topic.toHex() + peerId.toHex());
+    this._subscribedMessages = new ComplexSet2(({ peerId }) => peerId.toHex());
+    this.messageStreams = new ComplexMap2((key) => key.toHex());
+    this.reconciled = new Event();
+  }
+  async safeCloseStreams() {
+    const streams = [
+      ...this._swarmStreams.values()
+    ].concat([
+      ...this.messageStreams.values()
+    ]);
+    this._swarmStreams.clear();
+    this.messageStreams.clear();
+    const failureCount = (await safeAwaitAll(streams, (s) => s.close())).length;
+    return {
+      failureCount
+    };
+  }
+  join({ topic, peerId }) {
+    this._joinedTopics.add({
+      topic,
+      peerId
+    });
+  }
+  leave({ topic, peerId }) {
+    void this._swarmStreams.get({
+      topic,
+      peerId
+    })?.close();
+    this._swarmStreams.delete({
+      topic,
+      peerId
+    });
+    this._joinedTopics.delete({
+      topic,
+      peerId
+    });
+  }
+  subscribeMessages(peerId) {
+    this._subscribedMessages.add({
+      peerId
+    });
+  }
+  unsubscribeMessages(peerId) {
+    log2("unsubscribing from messages", {
+      peerId
+    }, {
+      F: __dxlog_file2,
+      L: 76,
+      S: this,
+      C: (f, a) => f(...a)
+    });
+    this._subscribedMessages.delete({
+      peerId
+    });
+    void this.messageStreams.get(peerId)?.close();
+    this.messageStreams.delete(peerId);
+  }
+  async reconcile(ctx, client) {
+    await this._reconcileSwarmSubscriptions(ctx, client);
+    await this._reconcileMessageSubscriptions(ctx, client);
+    this.reconciled.emit();
+  }
+  async _reconcileSwarmSubscriptions(ctx, client) {
+    for (const { topic, peerId } of this._swarmStreams.keys()) {
+      if (this._joinedTopics.has({
+        topic,
+        peerId
+      })) {
+        continue;
+      }
+      void this._swarmStreams.get({
+        topic,
+        peerId
+      })?.close();
+      this._swarmStreams.delete({
+        topic,
+        peerId
+      });
+    }
+    for (const { topic, peerId } of this._joinedTopics.values()) {
+      if (this._swarmStreams.has({
+        topic,
+        peerId
+      })) {
+        continue;
+      }
+      const swarmStream = await asyncTimeout(cancelWithContext(ctx, client.join({
+        topic,
+        peerId
+      })), 5e3);
+      swarmStream.subscribe(async (swarmEvent) => {
+        if (this._joinedTopics.has({
+          topic,
+          peerId
+        })) {
+          log2("swarm event", {
+            swarmEvent
+          }, {
+            F: __dxlog_file2,
+            L: 112,
+            S: this,
+            C: (f, a) => f(...a)
+          });
+          await this._onSwarmEvent({
+            topic,
+            swarmEvent
+          });
+        }
+      });
+      this._swarmStreams.set({
+        topic,
+        peerId
+      }, swarmStream);
+    }
+  }
+  async _reconcileMessageSubscriptions(ctx, client) {
+    for (const peerId of this.messageStreams.keys()) {
+      if (this._subscribedMessages.has({
+        peerId
+      })) {
+        continue;
+      }
+      void this.messageStreams.get(peerId)?.close();
+      this.messageStreams.delete(peerId);
+    }
+    for (const { peerId } of this._subscribedMessages.values()) {
+      if (this.messageStreams.has(peerId)) {
+        continue;
+      }
+      const messageStream = await asyncTimeout(cancelWithContext(ctx, client.receiveMessages(peerId)), 5e3);
+      messageStream.subscribe(async (signalMessage) => {
+        if (this._subscribedMessages.has({
+          peerId
+        })) {
+          const message = {
+            author: PublicKey2.from(signalMessage.author),
+            recipient: PublicKey2.from(signalMessage.recipient),
+            payload: signalMessage.payload
+          };
+          await this._onMessage(message);
+        }
+      });
+      this.messageStreams.set(peerId, messageStream);
+    }
+  }
+};
 
 // packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts
 import WebSocket from "isomorphic-ws";
-import { scheduleTaskInterval as scheduleTaskInterval2, Trigger } from "@dxos/async";
+import { scheduleTaskInterval as scheduleTaskInterval2, TimeoutError as TimeoutError2, Trigger } from "@dxos/async";
 import { Context as Context2 } from "@dxos/context";
 import { invariant as invariant2 } from "@dxos/invariant";
-import { PublicKey as PublicKey2 } from "@dxos/keys";
-import { log as log2 } from "@dxos/log";
-import { schema as schema2, trace as trace2 } from "@dxos/protocols";
+import { PublicKey as PublicKey3 } from "@dxos/keys";
+import { log as log3 } from "@dxos/log";
+import { schema as schema2, trace as trace5 } from "@dxos/protocols";
 import { createProtoRpcPeer } from "@dxos/rpc";
-var __dxlog_file2 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
+
+// packages/core/mesh/messaging/src/signal-client/signal-rpc-client-monitor.ts
+import { trace as trace4 } from "@dxos/tracing";
+var SignalRpcClientMonitor = class {
+  recordClientCloseFailure(params) {
+    trace4.metrics.increment("mesh.signal.signal-rpc-client.close-failure", 1, {
+      tags: {
+        reason: params.failureReason
+      }
+    });
+  }
+};
+
+// packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts
+var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-rpc-client.ts";
 var SIGNAL_KEEPALIVE_INTERVAL = 1e4;
 var SignalRPCClient = class {
   constructor({ url, callbacks = {} }) {
     this._connectTrigger = new Trigger();
     this._closed = false;
     this._closeComplete = new Trigger();
-    const traceId = PublicKey2.random().toHex();
-    log2.trace("dxos.mesh.signal-rpc-client.constructor", trace2.begin({
+    this._monitor = new SignalRpcClientMonitor();
+    const traceId = PublicKey3.random().toHex();
+    log3.trace("dxos.mesh.signal-rpc-client.constructor", trace5.begin({
       id: traceId
     }), {
-      F: __dxlog_file2,
-      L: 55,
+      F: __dxlog_file3,
+      L: 60,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -413,9 +718,9 @@ var SignalRPCClient = class {
           try {
             this._socket.send(msg);
           } catch (err) {
-            log2.warn("send error", err, {
-              F: __dxlog_file2,
-              L: 74,
+            log3.warn("send error", err, {
+              F: __dxlog_file3,
+              L: 79,
               S: this,
               C: (f, a) => f(...a)
             });
@@ -438,9 +743,13 @@ var SignalRPCClient = class {
     this._socket.onopen = async () => {
       try {
         await this._rpc.open();
-        log2(`RPC open ${this._url}`, void 0, {
-          F: __dxlog_file2,
-          L: 95,
+        if (this._closed) {
+          await this._safeCloseRpc();
+          return;
+        }
+        log3(`RPC open ${this._url}`, void 0, {
+          F: __dxlog_file3,
+          L: 104,
           S: this,
           C: (f, a) => f(...a)
         });
@@ -452,12 +761,14 @@ var SignalRPCClient = class {
         }, SIGNAL_KEEPALIVE_INTERVAL);
       } catch (err) {
         this._callbacks.onError?.(err);
+        this._socket.close();
+        this._closed = true;
       }
     };
     this._socket.onclose = async () => {
-      log2(`Disconnected ${this._url}`, void 0, {
-        F: __dxlog_file2,
-        L: 116,
+      log3(`Disconnected ${this._url}`, void 0, {
+        F: __dxlog_file3,
+        L: 127,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -467,89 +778,73 @@ var SignalRPCClient = class {
     };
     this._socket.onerror = async (event) => {
       if (this._closed) {
+        this._socket.close();
         return;
       }
-      this._callbacks.onError?.(event.error ?? new Error(event.message));
-      this._connectTrigger.reset();
-      try {
-        await this._rpc?.close();
-      } catch (err) {
-        log2.catch(err, void 0, {
-          F: __dxlog_file2,
-          L: 134,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-      }
       this._closed = true;
-      log2.warn(event.message ?? "Socket error", {
+      this._callbacks.onError?.(event.error ?? new Error(event.message));
+      await this._safeCloseRpc();
+      log3.warn(`Socket ${event.type ?? "unknown"} error`, {
+        message: event.message,
         url: this._url
       }, {
-        F: __dxlog_file2,
-        L: 138,
+        F: __dxlog_file3,
+        L: 143,
         S: this,
         C: (f, a) => f(...a)
       });
     };
-    log2.trace("dxos.mesh.signal-rpc-client.constructor", trace2.end({
+    log3.trace("dxos.mesh.signal-rpc-client.constructor", trace5.end({
       id: traceId
     }), {
-      F: __dxlog_file2,
-      L: 141,
+      F: __dxlog_file3,
+      L: 146,
       S: this,
       C: (f, a) => f(...a)
     });
   }
   async close() {
-    await this._keepaliveCtx?.dispose();
+    if (this._closed) {
+      return;
+    }
     this._closed = true;
+    await this._keepaliveCtx?.dispose();
     try {
-      await this._rpc?.close();
-      if (this._socket?.readyState === WebSocket.OPEN || this._socket?.readyState === WebSocket.CONNECTING) {
+      await this._safeCloseRpc();
+      if (this._socket.readyState === WebSocket.OPEN || this._socket.readyState === WebSocket.CONNECTING) {
         this._socket.close();
       }
       await this._closeComplete.wait({
         timeout: 1e3
       });
     } catch (err) {
-      log2.warn("close error", err, {
-        F: __dxlog_file2,
-        L: 156,
-        S: this,
-        C: (f, a) => f(...a)
+      const failureReason = err instanceof TimeoutError2 ? "timeout" : err?.constructor?.name ?? "unknown";
+      this._monitor.recordClientCloseFailure({
+        failureReason
       });
     }
   }
   async join({ topic, peerId }) {
-    log2("join", {
+    log3("join", {
       topic,
       peerId,
       metadata: this._callbacks?.getMetadata?.()
     }, {
-      F: __dxlog_file2,
-      L: 161,
+      F: __dxlog_file3,
+      L: 172,
       S: this,
       C: (f, a) => f(...a)
     });
-    await this._connectTrigger.wait();
     invariant2(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 163,
+      F: __dxlog_file3,
+      L: 173,
       S: this,
       A: [
         "!this._closed",
         "'SignalRPCClient is closed'"
       ]
     });
-    invariant2(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 164,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
+    await this._connectTrigger.wait();
     const swarmStream = this._rpc.rpc.Signal.join({
       swarm: topic.asUint8Array(),
       peer: peerId.asUint8Array(),
@@ -559,17 +854,17 @@ var SignalRPCClient = class {
     return swarmStream;
   }
   async receiveMessages(peerId) {
-    log2("receiveMessages", {
+    log3("receiveMessages", {
       peerId
     }, {
-      F: __dxlog_file2,
-      L: 175,
+      F: __dxlog_file3,
+      L: 185,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant2(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 176,
+      F: __dxlog_file3,
+      L: 186,
       S: this,
       A: [
         "!this._closed",
@@ -577,15 +872,6 @@ var SignalRPCClient = class {
       ]
     });
     await this._connectTrigger.wait();
-    invariant2(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 178,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
     const messageStream = this._rpc.rpc.Signal.receiveMessages({
       peer: peerId.asUint8Array()
     });
@@ -593,20 +879,20 @@ var SignalRPCClient = class {
     return messageStream;
   }
   async sendMessage({ author, recipient, payload }) {
-    log2("sendMessage", {
+    log3("sendMessage", {
       author,
       recipient,
       payload,
       metadata: this._callbacks?.getMetadata?.()
     }, {
-      F: __dxlog_file2,
-      L: 187,
+      F: __dxlog_file3,
+      L: 196,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant2(!this._closed, "SignalRPCClient is closed", {
-      F: __dxlog_file2,
-      L: 188,
+      F: __dxlog_file3,
+      L: 197,
       S: this,
       A: [
         "!this._closed",
@@ -614,15 +900,6 @@ var SignalRPCClient = class {
       ]
     });
     await this._connectTrigger.wait();
-    invariant2(this._rpc, "Rpc is not initialized", {
-      F: __dxlog_file2,
-      L: 190,
-      S: this,
-      A: [
-        "this._rpc",
-        "'Rpc is not initialized'"
-      ]
-    });
     await this._rpc.rpc.Signal.sendMessage({
       author: author.asUint8Array(),
       recipient: recipient.asUint8Array(),
@@ -630,53 +907,59 @@ var SignalRPCClient = class {
       metadata: this._callbacks?.getMetadata?.()
     });
   }
+  async _safeCloseRpc() {
+    try {
+      this._connectTrigger.reset();
+      await this._rpc.close();
+    } catch (err) {
+      log3.catch(err, void 0, {
+        F: __dxlog_file3,
+        L: 212,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
+  }
 };
 
 // packages/core/mesh/messaging/src/signal-client/signal-client.ts
-var __dxlog_file3 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
+var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-client/signal-client.ts";
 var DEFAULT_RECONNECT_TIMEOUT = 100;
 var MAX_RECONNECT_TIMEOUT = 5e3;
 var ERROR_RECONCILE_DELAY = 1e3;
 var RECONCILE_INTERVAL = 5e3;
-var SignalClient = class {
+var SignalClient = class extends Resource {
   /**
   * @param _host Signal server websocket URL.
+  * @param onMessage called when a new message is received.
+  * @param onSwarmEvent called when a new swarm event is received.
+  * @param _getMetadata signal-message metadata provider, called for every message.
   */
-  constructor(_host, _onMessage, _onSwarmEvent, _getMetadata) {
+  constructor(_host, onMessage, onSwarmEvent, _getMetadata) {
+    super();
     this._host = _host;
-    this._onMessage = _onMessage;
-    this._onSwarmEvent = _onSwarmEvent;
     this._getMetadata = _getMetadata;
+    this._monitor = new SignalClientMonitor();
     this._state = SignalState.CLOSED;
-    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
-    this._connectionStarted = /* @__PURE__ */ new Date();
-    this._lastStateChange = /* @__PURE__ */ new Date();
+    this._lastReconciliationFailed = false;
     this._clientReady = new Trigger2();
-    this.statusChanged = new Event();
-    this.commandTrace = new Event();
-    this._swarmStreams = new ComplexMap2(({ topic, peerId }) => topic.toHex() + peerId.toHex());
-    this._joinedTopics = new ComplexSet2(({ topic, peerId }) => topic.toHex() + peerId.toHex());
-    this._messageStreams = new ComplexMap2((key) => key.toHex());
-    this._subscribedMessages = new ComplexSet2(({ peerId }) => peerId.toHex());
-    this._reconciled = new Event();
-    this._instanceId = PublicKey3.random().toHex();
-    this._performance = {
-      sentMessages: 0,
-      receivedMessages: 0,
-      reconnectCounter: 0,
-      joinCounter: 0,
-      leaveCounter: 0
-    };
+    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
+    this._instanceId = PublicKey4.random().toHex();
+    this.statusChanged = new Event2();
     if (!this._host.startsWith("wss://") && !this._host.startsWith("ws://")) {
       throw new Error(`Signal server requires a websocket URL. Provided: ${this._host}`);
     }
+    this.localState = new SignalLocalState((message) => {
+      this._monitor.recordMessageReceived(message);
+      return onMessage(message);
+    }, onSwarmEvent);
   }
-  async open() {
-    log3.trace("dxos.mesh.signal-client.open", trace3.begin({
+  async _open() {
+    log4.trace("dxos.mesh.signal-client.open", trace6.begin({
       id: this._instanceId
     }), {
-      F: __dxlog_file3,
-      L: 128,
+      F: __dxlog_file4,
+      L: 83,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -686,26 +969,33 @@ var SignalClient = class {
     ].includes(this._state)) {
       return;
     }
-    this._ctx = new Context3({
-      onError: (err) => {
-        if (this._state === SignalState.CLOSED || this._ctx?.disposed) {
-          return;
-        }
-        if (this._state === SignalState.CONNECTED) {
-          log3.warn("SignalClient error:", err, {
-            F: __dxlog_file3,
-            L: 140,
-            S: this,
-            C: (f, a) => f(...a)
-          });
-        }
-        this._scheduleReconcileAfterError();
-      }
-    });
+    this._setState(SignalState.CONNECTING);
     this._reconcileTask = new DeferredTask(this._ctx, async () => {
-      await this._reconcileSwarmSubscriptions();
-      await this._reconcileMessageSubscriptions();
-      this._reconciled.emit();
+      try {
+        await cancelWithContext2(this._connectionCtx, this._clientReady.wait({
+          timeout: 5e3
+        }));
+        invariant3(this._state === SignalState.CONNECTED, "Not connected to Signal Server", {
+          F: __dxlog_file4,
+          L: 93,
+          S: this,
+          A: [
+            "this._state === SignalState.CONNECTED",
+            "'Not connected to Signal Server'"
+          ]
+        });
+        await this.localState.reconcile(this._connectionCtx, this._client);
+        this._monitor.recordReconciliation({
+          success: true
+        });
+        this._lastReconciliationFailed = false;
+      } catch (err) {
+        this._lastReconciliationFailed = true;
+        this._monitor.recordReconciliation({
+          success: false
+        });
+        throw err;
+      }
     });
     scheduleTaskInterval3(this._ctx, async () => {
       if (this._state === SignalState.CONNECTED) {
@@ -713,23 +1003,46 @@ var SignalClient = class {
       }
     }, RECONCILE_INTERVAL);
     this._reconnectTask = new DeferredTask(this._ctx, async () => {
-      await this._reconnect();
+      try {
+        await this._reconnect();
+        this._monitor.recordReconnect({
+          success: true
+        });
+      } catch (err) {
+        this._monitor.recordReconnect({
+          success: false
+        });
+        throw err;
+      }
     });
-    this._setState(SignalState.CONNECTING);
     this._createClient();
-    log3.trace("dxos.mesh.signal-client.open", trace3.end({
+    log4.trace("dxos.mesh.signal-client.open", trace6.end({
       id: this._instanceId
     }), {
-      F: __dxlog_file3,
-      L: 169,
+      F: __dxlog_file4,
+      L: 126,
       S: this,
       C: (f, a) => f(...a)
     });
   }
-  async close() {
-    log3("closing...", void 0, {
-      F: __dxlog_file3,
-      L: 173,
+  async _catch(err) {
+    if (this._state === SignalState.CLOSED || this._ctx.disposed) {
+      return;
+    }
+    if (this._state === SignalState.CONNECTED && !this._lastReconciliationFailed) {
+      log4.warn("SignalClient error:", err, {
+        F: __dxlog_file4,
+        L: 135,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+    }
+    this._scheduleReconcileAfterError();
+  }
+  async _close() {
+    log4("closing...", void 0, {
+      F: __dxlog_file4,
+      L: 141,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -738,14 +1051,11 @@ var SignalClient = class {
     ].includes(this._state)) {
       return;
     }
-    await this._ctx?.dispose();
-    this._clientReady.reset();
-    await this._client?.close();
-    this._client = void 0;
     this._setState(SignalState.CLOSED);
-    log3("closed", void 0, {
-      F: __dxlog_file3,
-      L: 184,
+    await this._safeResetClient();
+    log4("closed", void 0, {
+      F: __dxlog_file4,
+      L: 149,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -756,228 +1066,181 @@ var SignalClient = class {
       state: this._state,
       error: this._lastError?.message,
       reconnectIn: this._reconnectAfter,
-      connectionStarted: this._connectionStarted,
-      lastStateChange: this._lastStateChange
+      ...this._monitor.getRecordedTimestamps()
     };
   }
-  async join({ topic, peerId }) {
-    log3("joining", {
-      topic,
-      peerId
+  async join(args) {
+    log4("joining", {
+      topic: args.topic,
+      peerId: args.peerId
     }, {
-      F: __dxlog_file3,
-      L: 199,
+      F: __dxlog_file4,
+      L: 163,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._performance.joinCounter++;
-    this._joinedTopics.add({
-      topic,
-      peerId
-    });
-    this._reconcileTask.schedule();
+    this._monitor.recordJoin();
+    this.localState.join(args);
+    this._reconcileTask?.schedule();
   }
-  async leave({ topic, peerId }) {
-    this._performance.leaveCounter++;
-    log3("leaving", {
-      topic,
-      peerId
+  async leave(args) {
+    log4("leaving", {
+      topic: args.topic,
+      peerId: args.peerId
     }, {
-      F: __dxlog_file3,
-      L: 207,
+      F: __dxlog_file4,
+      L: 170,
       S: this,
       C: (f, a) => f(...a)
     });
-    void this._swarmStreams.get({
-      topic,
-      peerId
-    })?.close();
-    this._swarmStreams.delete({
-      topic,
-      peerId
-    });
-    this._joinedTopics.delete({
-      topic,
-      peerId
-    });
+    this._monitor.recordLeave();
+    this.localState.leave(args);
   }
   async sendMessage(msg) {
-    this._performance.sentMessages++;
-    await this._clientReady.wait();
-    invariant3(this._state === SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 216,
-      S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
+    return this._monitor.recordMessageSending(msg, async () => {
+      await this._clientReady.wait();
+      invariant3(this._state === SignalState.CONNECTED, "Not connected to Signal Server", {
+        F: __dxlog_file4,
+        L: 178,
+        S: this,
+        A: [
+          "this._state === SignalState.CONNECTED",
+          "'Not connected to Signal Server'"
+        ]
+      });
+      await this._client.sendMessage(msg);
     });
-    await this._client.sendMessage(msg);
   }
   async subscribeMessages(peerId) {
-    log3("subscribing to messages", {
+    log4("subscribing to messages", {
       peerId
     }, {
-      F: __dxlog_file3,
-      L: 221,
+      F: __dxlog_file4,
+      L: 184,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._subscribedMessages.add({
-      peerId
-    });
-    this._reconcileTask.schedule();
+    this.localState.subscribeMessages(peerId);
+    this._reconcileTask?.schedule();
   }
   async unsubscribeMessages(peerId) {
-    log3("unsubscribing from messages", {
+    log4("unsubscribing from messages", {
       peerId
     }, {
-      F: __dxlog_file3,
-      L: 227,
+      F: __dxlog_file4,
+      L: 190,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._subscribedMessages.delete({
-      peerId
-    });
-    void this._messageStreams.get(peerId)?.close();
-    this._messageStreams.delete(peerId);
+    this.localState.unsubscribeMessages(peerId);
   }
   _scheduleReconcileAfterError() {
-    scheduleTask2(this._ctx, () => {
-      this._reconcileTask.schedule();
-    }, ERROR_RECONCILE_DELAY);
-  }
-  _setState(newState) {
-    this._state = newState;
-    this._lastStateChange = /* @__PURE__ */ new Date();
-    log3("signal state changed", {
-      status: this.getStatus()
-    }, {
-      F: __dxlog_file3,
-      L: 246,
-      S: this,
-      C: (f, a) => f(...a)
-    });
-    this.statusChanged.emit(this.getStatus());
+    scheduleTask2(this._ctx, () => this._reconcileTask.schedule(), ERROR_RECONCILE_DELAY);
   }
   _createClient() {
-    log3("creating client", {
+    log4("creating client", {
       host: this._host,
       state: this._state
     }, {
-      F: __dxlog_file3,
-      L: 251,
+      F: __dxlog_file4,
+      L: 199,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant3(!this._client, "Client already created", {
-      F: __dxlog_file3,
-      L: 252,
+      F: __dxlog_file4,
+      L: 200,
       S: this,
       A: [
         "!this._client",
         "'Client already created'"
       ]
     });
-    this._connectionStarted = /* @__PURE__ */ new Date();
+    this._monitor.recordConnectionStartTime();
     this._connectionCtx = this._ctx.derive();
     this._connectionCtx.onDispose(async () => {
-      log3("connection context disposed", void 0, {
-        F: __dxlog_file3,
-        L: 259,
+      log4("connection context disposed", void 0, {
+        F: __dxlog_file4,
+        L: 207,
         S: this,
         C: (f, a) => f(...a)
       });
-      await Promise.all(Array.from(this._swarmStreams.values()).map((stream) => stream.close()));
-      await Promise.all(Array.from(this._messageStreams.values()).map((stream) => stream.close()));
-      this._swarmStreams.clear();
-      this._messageStreams.clear();
+      const { failureCount } = await this.localState.safeCloseStreams();
+      this._monitor.recordStreamCloseErrors(failureCount);
     });
     try {
-      this._client = new SignalRPCClient({
+      const client = new SignalRPCClient({
         url: this._host,
         callbacks: {
           onConnected: () => {
-            log3("socket connected", void 0, {
-              F: __dxlog_file3,
-              L: 271,
-              S: this,
-              C: (f, a) => f(...a)
-            });
-            this._lastError = void 0;
-            this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
-            this._setState(SignalState.CONNECTED);
-            this._clientReady.wake();
-            this._reconcileTask.schedule();
+            if (client === this._client) {
+              log4("socket connected", void 0, {
+                F: __dxlog_file4,
+                L: 218,
+                S: this,
+                C: (f, a) => f(...a)
+              });
+              this._onConnected();
+            }
           },
           onDisconnected: () => {
-            log3("socket disconnected", {
+            if (client !== this._client) {
+              return;
+            }
+            log4("socket disconnected", {
               state: this._state
             }, {
-              F: __dxlog_file3,
-              L: 280,
+              F: __dxlog_file4,
+              L: 227,
               S: this,
               C: (f, a) => f(...a)
             });
             if (this._state === SignalState.ERROR) {
               this._setState(SignalState.DISCONNECTED);
-              return;
-            }
-            if (this._state !== SignalState.CONNECTED && this._state !== SignalState.CONNECTING) {
-              this._incrementReconnectTimeout();
+            } else {
+              this._onDisconnected();
             }
-            this._setState(SignalState.DISCONNECTED);
-            this._reconnectTask.schedule();
           },
           onError: (error) => {
-            log3("socket error", {
-              error,
-              state: this._state
-            }, {
-              F: __dxlog_file3,
-              L: 295,
-              S: this,
-              C: (f, a) => f(...a)
-            });
-            this._lastError = error;
-            if (this._state !== SignalState.CONNECTED && this._state !== SignalState.CONNECTING) {
-              this._incrementReconnectTimeout();
+            if (client === this._client) {
+              log4("socket error", {
+                error,
+                state: this._state
+              }, {
+                F: __dxlog_file4,
+                L: 239,
+                S: this,
+                C: (f, a) => f(...a)
+              });
+              this._onDisconnected({
+                error
+              });
             }
-            this._setState(SignalState.ERROR);
-            this._reconnectTask.schedule();
           },
           getMetadata: this._getMetadata
         }
       });
-    } catch (err) {
-      if (this._state !== SignalState.CONNECTED && this._state !== SignalState.CONNECTING) {
-        this._incrementReconnectTimeout();
-      }
-      this._lastError = err;
-      this._setState(SignalState.DISCONNECTED);
-      this._reconnectTask.schedule();
+      this._client = client;
+    } catch (error) {
+      this._client = void 0;
+      this._onDisconnected({
+        error
+      });
     }
   }
-  _incrementReconnectTimeout() {
-    this._reconnectAfter *= 2;
-    this._reconnectAfter = Math.min(this._reconnectAfter, MAX_RECONNECT_TIMEOUT);
-  }
   async _reconnect() {
-    log3(`reconnecting in ${this._reconnectAfter}ms`, {
+    log4(`reconnecting in ${this._reconnectAfter}ms`, {
       state: this._state
     }, {
-      F: __dxlog_file3,
-      L: 323,
+      F: __dxlog_file4,
+      L: 254,
       S: this,
       C: (f, a) => f(...a)
     });
-    this._performance.reconnectCounter++;
     if (this._state === SignalState.RECONNECTING) {
-      log3.warn("Signal api already reconnecting.", void 0, {
-        F: __dxlog_file3,
-        L: 327,
+      log4.info("Signal api already reconnecting.", void 0, {
+        F: __dxlog_file4,
+        L: 257,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -986,149 +1249,96 @@ var SignalClient = class {
     if (this._state === SignalState.CLOSED) {
       return;
     }
-    this._clientReady.reset();
-    await this._connectionCtx?.dispose();
-    this._client?.close().catch(() => {
-    });
-    this._client = void 0;
-    await cancelWithContext(this._ctx, sleep(this._reconnectAfter));
     this._setState(SignalState.RECONNECTING);
+    await this._safeResetClient();
+    await cancelWithContext2(this._ctx, sleep(this._reconnectAfter));
     this._createClient();
   }
-  async _reconcileSwarmSubscriptions() {
-    await asyncTimeout(cancelWithContext(this._connectionCtx, this._clientReady.wait()), 5e3);
-    const client = this._client;
-    invariant3(this._state === SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 352,
-      S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
-    });
-    for (const { topic, peerId } of this._swarmStreams.keys()) {
-      if (this._joinedTopics.has({
-        topic,
-        peerId
-      })) {
-        continue;
-      }
-      void this._swarmStreams.get({
-        topic,
-        peerId
-      })?.close();
-      this._swarmStreams.delete({
-        topic,
-        peerId
-      });
+  _onConnected() {
+    this._lastError = void 0;
+    this._lastReconciliationFailed = false;
+    this._reconnectAfter = DEFAULT_RECONNECT_TIMEOUT;
+    this._setState(SignalState.CONNECTED);
+    this._clientReady.wake();
+    this._reconcileTask.schedule();
+  }
+  _onDisconnected(options) {
+    this._updateReconnectTimeout();
+    if (this._state === SignalState.CLOSED) {
+      return;
     }
-    for (const { topic, peerId } of this._joinedTopics.values()) {
-      if (this._swarmStreams.has({
-        topic,
-        peerId
-      })) {
-        continue;
-      }
-      const swarmStream = await asyncTimeout(cancelWithContext(this._connectionCtx, client.join({
-        topic,
-        peerId
-      })), 5e3);
-      swarmStream.subscribe(async (swarmEvent) => {
-        log3("swarm event", {
-          swarmEvent
-        }, {
-          F: __dxlog_file3,
-          L: 379,
-          S: this,
-          C: (f, a) => f(...a)
-        });
-        await this._onSwarmEvent({
-          topic,
-          swarmEvent
-        });
-      });
-      this._swarmStreams.set({
-        topic,
-        peerId
-      }, swarmStream);
+    if (options?.error) {
+      this._lastError = options.error;
+      this._setState(SignalState.ERROR);
+    } else {
+      this._setState(SignalState.DISCONNECTED);
     }
+    this._reconnectTask.schedule();
   }
-  async _reconcileMessageSubscriptions() {
-    await asyncTimeout(cancelWithContext(this._connectionCtx, this._clientReady.wait()), 5e3);
-    const client = this._client;
-    invariant3(this._state === SignalState.CONNECTED, "Not connected to Signal Server", {
-      F: __dxlog_file3,
-      L: 392,
+  _setState(newState) {
+    this._state = newState;
+    this._monitor.recordStateChangeTime();
+    log4("signal state changed", {
+      status: this.getStatus()
+    }, {
+      F: __dxlog_file4,
+      L: 298,
       S: this,
-      A: [
-        "this._state === SignalState.CONNECTED",
-        "'Not connected to Signal Server'"
-      ]
+      C: (f, a) => f(...a)
     });
-    for (const peerId of this._messageStreams.keys()) {
-      if (this._subscribedMessages.has({
-        peerId
-      })) {
-        continue;
-      }
-      void this._messageStreams.get(peerId)?.close();
-      this._messageStreams.delete(peerId);
-    }
-    for (const { peerId } of this._subscribedMessages.values()) {
-      if (this._messageStreams.has(peerId)) {
-        continue;
-      }
-      const messageStream = await asyncTimeout(cancelWithContext(this._connectionCtx, client.receiveMessages(peerId)), 5e3);
-      messageStream.subscribe(async (message) => {
-        this._performance.receivedMessages++;
-        await this._onMessage({
-          author: PublicKey3.from(message.author),
-          recipient: PublicKey3.from(message.recipient),
-          payload: message.payload
-        });
-      });
-      this._messageStreams.set(peerId, messageStream);
+    this.statusChanged.emit(this.getStatus());
+  }
+  _updateReconnectTimeout() {
+    if (this._state !== SignalState.CONNECTED && this._state !== SignalState.CONNECTING) {
+      this._reconnectAfter *= 2;
+      this._reconnectAfter = Math.min(this._reconnectAfter, MAX_RECONNECT_TIMEOUT);
     }
   }
+  async _safeResetClient() {
+    await this._connectionCtx?.dispose();
+    this._connectionCtx = void 0;
+    this._clientReady.reset();
+    await this._client?.close().catch(() => {
+    });
+    this._client = void 0;
+  }
 };
 
 // packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts
-import { Event as Event2, Trigger as Trigger3 } from "@dxos/async";
-import { Context as Context4 } from "@dxos/context";
+import { Event as Event3, Trigger as Trigger3 } from "@dxos/async";
+import { Context as Context3 } from "@dxos/context";
 import { invariant as invariant4 } from "@dxos/invariant";
-import { PublicKey as PublicKey4 } from "@dxos/keys";
-import { log as log4 } from "@dxos/log";
+import { PublicKey as PublicKey5 } from "@dxos/keys";
+import { log as log5 } from "@dxos/log";
 import { schema as schema3 } from "@dxos/protocols";
 import { ComplexMap as ComplexMap3, ComplexSet as ComplexSet3 } from "@dxos/util";
-var __dxlog_file4 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
+var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/memory-signal-manager.ts";
 var MemorySignalManagerContext = class {
   constructor() {
     // Swarm messages.
-    this.swarmEvent = new Event2();
+    this.swarmEvent = new Event3();
     // Mapping from topic to set of peers.
-    this.swarms = new ComplexMap3(PublicKey4.hash);
+    this.swarms = new ComplexMap3(PublicKey5.hash);
     // Map of connections for each peer for signaling.
-    this.connections = new ComplexMap3(PublicKey4.hash);
+    this.connections = new ComplexMap3(PublicKey5.hash);
   }
 };
 var MemorySignalManager = class {
   constructor(_context) {
     this._context = _context;
-    this.statusChanged = new Event2();
-    this.commandTrace = new Event2();
-    this.swarmEvent = new Event2();
-    this.onMessage = new Event2();
+    this.statusChanged = new Event3();
+    this.swarmEvent = new Event3();
+    this.onMessage = new Event3();
     this._joinedSwarms = new ComplexSet3(({ topic, peerId }) => topic.toHex() + peerId.toHex());
     this._freezeTrigger = new Trigger3().wake();
-    this._ctx = new Context4();
+    this._ctx = new Context3();
     this._ctx.onDispose(this._context.swarmEvent.on((data) => this.swarmEvent.emit(data)));
   }
   async open() {
     if (!this._ctx.disposed) {
       return;
     }
-    this._ctx = new Context4();
+    this._ctx = new Context3();
     this._ctx.onDispose(this._context.swarmEvent.on((data) => this.swarmEvent.emit(data)));
     await Promise.all([
       ...this._joinedSwarms.values()
@@ -1152,8 +1362,8 @@ var MemorySignalManager = class {
   }
   async join({ topic, peerId }) {
     invariant4(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 102,
+      F: __dxlog_file5,
+      L: 100,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1165,7 +1375,7 @@ var MemorySignalManager = class {
       peerId
     });
     if (!this._context.swarms.has(topic)) {
-      this._context.swarms.set(topic, new ComplexSet3(PublicKey4.hash));
+      this._context.swarms.set(topic, new ComplexSet3(PublicKey5.hash));
     }
     this._context.swarms.get(topic).add(peerId);
     this._context.swarmEvent.emit({
@@ -1193,8 +1403,8 @@ var MemorySignalManager = class {
   }
   async leave({ topic, peerId }) {
     invariant4(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 138,
+      F: __dxlog_file5,
+      L: 136,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1206,7 +1416,7 @@ var MemorySignalManager = class {
       peerId
     });
     if (!this._context.swarms.has(topic)) {
-      this._context.swarms.set(topic, new ComplexSet3(PublicKey4.hash));
+      this._context.swarms.set(topic, new ComplexSet3(PublicKey5.hash));
     }
     this._context.swarms.get(topic).delete(peerId);
     const swarmEvent = {
@@ -1220,19 +1430,19 @@ var MemorySignalManager = class {
     });
   }
   async sendMessage({ author, recipient, payload }) {
-    log4("send message", {
+    log5("send message", {
       author,
       recipient,
       ...dec(payload)
     }, {
-      F: __dxlog_file4,
-      L: 158,
+      F: __dxlog_file5,
+      L: 156,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant4(recipient, void 0, {
-      F: __dxlog_file4,
-      L: 160,
+      F: __dxlog_file5,
+      L: 158,
       S: this,
       A: [
         "recipient",
@@ -1240,8 +1450,8 @@ var MemorySignalManager = class {
       ]
     });
     invariant4(!this._ctx.disposed, "Closed", {
-      F: __dxlog_file4,
-      L: 161,
+      F: __dxlog_file5,
+      L: 159,
       S: this,
       A: [
         "!this._ctx.disposed",
@@ -1251,24 +1461,24 @@ var MemorySignalManager = class {
     await this._freezeTrigger.wait();
     const remote = this._context.connections.get(recipient);
     if (!remote) {
-      log4.warn("recipient is not subscribed for messages", {
+      log5.warn("recipient is not subscribed for messages", {
         author,
         recipient
       }, {
-        F: __dxlog_file4,
-        L: 167,
+        F: __dxlog_file5,
+        L: 165,
         S: this,
         C: (f, a) => f(...a)
       });
       return;
     }
     if (remote._ctx.disposed) {
-      log4.warn("recipient is disposed", {
+      log5.warn("recipient is disposed", {
         author,
         recipient
       }, {
-        F: __dxlog_file4,
-        L: 172,
+        F: __dxlog_file5,
+        L: 170,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1276,24 +1486,24 @@ var MemorySignalManager = class {
     }
     remote._freezeTrigger.wait().then(() => {
       if (remote._ctx.disposed) {
-        log4.warn("recipient is disposed", {
+        log5.warn("recipient is disposed", {
           author,
           recipient
         }, {
-          F: __dxlog_file4,
-          L: 180,
+          F: __dxlog_file5,
+          L: 178,
           S: this,
           C: (f, a) => f(...a)
         });
         return;
       }
-      log4("receive message", {
+      log5("receive message", {
         author,
         recipient,
         ...dec(payload)
       }, {
-        F: __dxlog_file4,
-        L: 184,
+        F: __dxlog_file5,
+        L: 182,
         S: this,
         C: (f, a) => f(...a)
       });
@@ -1303,33 +1513,33 @@ var MemorySignalManager = class {
         payload
       });
     }).catch((err) => {
-      log4.error("error while waiting for freeze", {
+      log5.error("error while waiting for freeze", {
         err
       }, {
-        F: __dxlog_file4,
-        L: 189,
+        F: __dxlog_file5,
+        L: 187,
         S: this,
         C: (f, a) => f(...a)
       });
     });
   }
   async subscribeMessages(peerId) {
-    log4("subscribing", {
+    log5("subscribing", {
       peerId
     }, {
-      F: __dxlog_file4,
-      L: 194,
+      F: __dxlog_file5,
+      L: 192,
       S: this,
       C: (f, a) => f(...a)
     });
     this._context.connections.set(peerId, this);
   }
   async unsubscribeMessages(peerId) {
-    log4("unsubscribing", {
+    log5("unsubscribing", {
       peerId
     }, {
-      F: __dxlog_file4,
-      L: 199,
+      F: __dxlog_file5,
+      L: 197,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1357,12 +1567,31 @@ var dec = (payload) => {
 };
 
 // packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts
-import { Event as Event3, sleep as sleep2, synchronized } from "@dxos/async";
-import { Context as Context5 } from "@dxos/context";
+import { Event as Event4, sleep as sleep2, synchronized } from "@dxos/async";
+import { Context as Context4 } from "@dxos/context";
 import { invariant as invariant5 } from "@dxos/invariant";
-import { PublicKey as PublicKey5 } from "@dxos/keys";
-import { log as log5 } from "@dxos/log";
-import { RateLimitExceededError, TimeoutError as TimeoutError2, trace as trace4 } from "@dxos/protocols";
+import { PublicKey as PublicKey6 } from "@dxos/keys";
+import { log as log6 } from "@dxos/log";
+import { RateLimitExceededError, TimeoutError as TimeoutError3, trace as trace8 } from "@dxos/protocols";
+import { BitField, safeAwaitAll as safeAwaitAll2 } from "@dxos/util";
+
+// packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager-monitor.ts
+import { trace as trace7 } from "@dxos/tracing";
+var WebsocketSignalManagerMonitor = class {
+  recordRateLimitExceeded() {
+    trace7.metrics.increment("mesh.signal.signal-manager.rate-limit-hit", 1);
+  }
+  recordServerFailure(params) {
+    trace7.metrics.increment("mesh.signal.signal-manager.server-failure", 1, {
+      tags: {
+        server: params.serverName,
+        restarted: params.willRestart
+      }
+    });
+  }
+};
+
+// packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts
 function _ts_decorate(decorators, target, key, desc) {
   var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
   if (typeof Reflect === "object" && typeof Reflect.decorate === "function")
@@ -1373,7 +1602,7 @@ function _ts_decorate(decorators, target, key, desc) {
         r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
-var __dxlog_file5 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
+var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/websocket-signal-manager.ts";
 var MAX_SERVER_FAILURES = 5;
 var WSS_SIGNAL_SERVER_REBOOT_DELAY = 3e3;
 var WebsocketSignalManager = class {
@@ -1381,18 +1610,18 @@ var WebsocketSignalManager = class {
     this._hosts = _hosts;
     this._getMetadata = _getMetadata;
     this._servers = /* @__PURE__ */ new Map();
+    this._monitor = new WebsocketSignalManagerMonitor();
     this._opened = false;
     this.failureCount = /* @__PURE__ */ new Map();
-    this.statusChanged = new Event3();
-    this.commandTrace = new Event3();
-    this.swarmEvent = new Event3();
-    this.onMessage = new Event3();
-    this._instanceId = PublicKey5.random().toHex();
-    log5("Created WebsocketSignalManager", {
+    this.statusChanged = new Event4();
+    this.swarmEvent = new Event4();
+    this.onMessage = new Event4();
+    this._instanceId = PublicKey6.random().toHex();
+    log6("Created WebsocketSignalManager", {
       hosts: this._hosts
     }, {
-      F: __dxlog_file5,
-      L: 51,
+      F: __dxlog_file6,
+      L: 58,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1404,39 +1633,37 @@ var WebsocketSignalManager = class {
       server.statusChanged.on(() => this.statusChanged.emit(this.getStatus()));
       this._servers.set(host.server, server);
       this.failureCount.set(host.server, 0);
-      server.commandTrace.on((trace5) => this.commandTrace.emit(trace5));
     }
+    this._failedServersBitfield = BitField.zeros(this._hosts.length);
   }
   async open() {
     if (this._opened) {
       return;
     }
-    log5("open signal manager", {
+    log6("open signal manager", {
       hosts: this._hosts
     }, {
-      F: __dxlog_file5,
-      L: 78,
+      F: __dxlog_file6,
+      L: 85,
       S: this,
       C: (f, a) => f(...a)
     });
-    log5.trace("dxos.mesh.websocket-signal-manager.open", trace4.begin({
+    log6.trace("dxos.mesh.websocket-signal-manager.open", trace8.begin({
       id: this._instanceId
     }), {
-      F: __dxlog_file5,
-      L: 79,
+      F: __dxlog_file6,
+      L: 86,
       S: this,
       C: (f, a) => f(...a)
     });
     this._initContext();
-    [
-      ...this._servers.values()
-    ].forEach((server) => server.open());
+    await safeAwaitAll2(this._servers.values(), (server) => server.open());
     this._opened = true;
-    log5.trace("dxos.mesh.websocket-signal-manager.open", trace4.end({
+    log6.trace("dxos.mesh.websocket-signal-manager.open", trace8.end({
       id: this._instanceId
     }), {
-      F: __dxlog_file5,
-      L: 87,
+      F: __dxlog_file6,
+      L: 93,
       S: this,
       C: (f, a) => f(...a)
     });
@@ -1447,20 +1674,20 @@ var WebsocketSignalManager = class {
     }
     this._opened = false;
     await this._ctx.dispose();
-    await Promise.all(Array.from(this._servers.values()).map((server) => server.close()));
+    await safeAwaitAll2(this._servers.values(), (server) => server.close());
   }
   async restartServer(serverName) {
-    log5("restarting server", {
+    log6("restarting server", {
       serverName
     }, {
-      F: __dxlog_file5,
-      L: 103,
+      F: __dxlog_file6,
+      L: 107,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "server already closed", {
-      F: __dxlog_file5,
-      L: 104,
+      F: __dxlog_file6,
+      L: 108,
       S: this,
       A: [
         "this._opened",
@@ -1469,8 +1696,8 @@ var WebsocketSignalManager = class {
     });
     const server = this._servers.get(serverName);
     invariant5(server, "server not found", {
-      F: __dxlog_file5,
-      L: 107,
+      F: __dxlog_file6,
+      L: 111,
       S: this,
       A: [
         "server",
@@ -1485,18 +1712,18 @@ var WebsocketSignalManager = class {
     return Array.from(this._servers.values()).map((server) => server.getStatus());
   }
   async join({ topic, peerId }) {
-    log5("join", {
+    log6("join", {
       topic,
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 120,
+      F: __dxlog_file6,
+      L: 124,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 121,
+      F: __dxlog_file6,
+      L: 125,
       S: this,
       A: [
         "this._opened",
@@ -1509,18 +1736,18 @@ var WebsocketSignalManager = class {
     }));
   }
   async leave({ topic, peerId }) {
-    log5("leaving", {
+    log6("leaving", {
       topic,
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 127,
+      F: __dxlog_file6,
+      L: 131,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 128,
+      F: __dxlog_file6,
+      L: 132,
       S: this,
       A: [
         "this._opened",
@@ -1533,89 +1760,115 @@ var WebsocketSignalManager = class {
     }));
   }
   async sendMessage({ author, recipient, payload }) {
-    log5("signal", {
+    log6("signal", {
       recipient
     }, {
-      F: __dxlog_file5,
-      L: 142,
+      F: __dxlog_file6,
+      L: 145,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 143,
+      F: __dxlog_file6,
+      L: 146,
       S: this,
       A: [
         "this._opened",
         "'Closed'"
       ]
     });
-    void this._forEachServer(async (server, serverName) => {
+    void this._forEachServer(async (server, serverName, index) => {
       void server.sendMessage({
         author,
         recipient,
         payload
-      }).catch((err) => {
+      }).then(() => this._clearServerFailedFlag(serverName, index)).catch((err) => {
         if (err instanceof RateLimitExceededError) {
-          log5.info("WSS rate limit exceeded", {
+          log6.info("WSS rate limit exceeded", {
             err
           }, {
-            F: __dxlog_file5,
-            L: 148,
+            F: __dxlog_file6,
+            L: 154,
             S: this,
             C: (f, a) => f(...a)
           });
-        } else if (err instanceof TimeoutError2 || err.constructor.name === "TimeoutError") {
-          log5.info("WSS sendMessage timeout", {
+          this._monitor.recordRateLimitExceeded();
+        } else if (err instanceof TimeoutError3 || err.constructor.name === "TimeoutError") {
+          log6.info("WSS sendMessage timeout", {
             err
           }, {
-            F: __dxlog_file5,
-            L: 150,
+            F: __dxlog_file6,
+            L: 157,
             S: this,
             C: (f, a) => f(...a)
           });
-          void this.checkServerFailure(serverName);
+          void this.checkServerFailure(serverName, index);
         } else {
-          log5.info(`error sending to ${serverName}`, {
+          log6.warn(`error sending to ${serverName}`, {
             err
           }, {
-            F: __dxlog_file5,
-            L: 153,
+            F: __dxlog_file6,
+            L: 160,
             S: this,
             C: (f, a) => f(...a)
           });
-          void this.checkServerFailure(serverName);
+          void this.checkServerFailure(serverName, index);
         }
       });
     });
   }
-  async checkServerFailure(serverName) {
+  async checkServerFailure(serverName, index) {
     const failureCount = this.failureCount.get(serverName) ?? 0;
-    if (failureCount > MAX_SERVER_FAILURES) {
-      log5.warn(`too many failures sending to ${serverName} (${failureCount} > ${MAX_SERVER_FAILURES}), restarting`, void 0, {
-        F: __dxlog_file5,
-        L: 164,
-        S: this,
-        C: (f, a) => f(...a)
-      });
+    const isRestartRequired = failureCount > MAX_SERVER_FAILURES;
+    this._monitor.recordServerFailure({
+      serverName,
+      willRestart: isRestartRequired
+    });
+    if (isRestartRequired) {
+      if (!BitField.get(this._failedServersBitfield, index)) {
+        log6.warn("too many failures for ws-server, restarting", {
+          serverName,
+          failureCount
+        }, {
+          F: __dxlog_file6,
+          L: 174,
+          S: this,
+          C: (f, a) => f(...a)
+        });
+        BitField.set(this._failedServersBitfield, index, true);
+      }
       await this.restartServer(serverName);
       this.failureCount.set(serverName, 0);
       return;
     }
     this.failureCount.set(serverName, (this.failureCount.get(serverName) ?? 0) + 1);
   }
+  _clearServerFailedFlag(serverName, index) {
+    if (BitField.get(this._failedServersBitfield, index)) {
+      log6.info("server connection restored", {
+        serverName
+      }, {
+        F: __dxlog_file6,
+        L: 187,
+        S: this,
+        C: (f, a) => f(...a)
+      });
+      BitField.set(this._failedServersBitfield, index, false);
+      this.failureCount.set(serverName, 0);
+    }
+  }
   async subscribeMessages(peerId) {
-    log5("subscribed for message stream", {
+    log6("subscribed for message stream", {
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 174,
+      F: __dxlog_file6,
+      L: 194,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 175,
+      F: __dxlog_file6,
+      L: 195,
       S: this,
       A: [
         "this._opened",
@@ -1625,17 +1878,17 @@ var WebsocketSignalManager = class {
     await this._forEachServer(async (server) => server.subscribeMessages(peerId));
   }
   async unsubscribeMessages(peerId) {
-    log5("subscribed for message stream", {
+    log6("subscribed for message stream", {
       peerId
     }, {
-      F: __dxlog_file5,
-      L: 181,
+      F: __dxlog_file6,
+      L: 201,
       S: this,
       C: (f, a) => f(...a)
     });
     invariant5(this._opened, "Closed", {
-      F: __dxlog_file5,
-      L: 182,
+      F: __dxlog_file6,
+      L: 202,
       S: this,
       A: [
         "this._opened",
@@ -1645,17 +1898,17 @@ var WebsocketSignalManager = class {
     await this._forEachServer(async (server) => server.unsubscribeMessages(peerId));
   }
   _initContext() {
-    this._ctx = new Context5({
-      onError: (err) => log5.catch(err, void 0, {
-        F: __dxlog_file5,
-        L: 189,
+    this._ctx = new Context4({
+      onError: (err) => log6.catch(err, void 0, {
+        F: __dxlog_file6,
+        L: 209,
         S: this,
         C: (f, a) => f(...a)
       })
     });
   }
   async _forEachServer(fn) {
-    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server]) => fn(server, serverName)));
+    return Promise.all(Array.from(this._servers.entries()).map(([serverName, server], idx) => fn(server, serverName, idx)));
   }
 };
 _ts_decorate([
@@ -1676,16 +1929,16 @@ _ts_decorate([
 
 // packages/core/mesh/messaging/src/signal-manager/utils.ts
 import { invariant as invariant6 } from "@dxos/invariant";
-import { log as log6 } from "@dxos/log";
+import { log as log7 } from "@dxos/log";
 import { DeviceKind } from "@dxos/protocols/proto/dxos/client/services";
-var __dxlog_file6 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/utils.ts";
+var __dxlog_file7 = "/home/runner/work/dxos/dxos/packages/core/mesh/messaging/src/signal-manager/utils.ts";
 var setIdentityTags = ({ identityService, devicesService, setTag }) => {
   identityService.queryIdentity().subscribe((idqr) => {
     if (!idqr?.identity?.identityKey) {
-      log6("empty response from identity service", {
+      log7("empty response from identity service", {
         idqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 21,
         S: void 0,
         C: (f, a) => f(...a)
@@ -1696,10 +1949,10 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
   });
   devicesService.queryDevices().subscribe((dqr) => {
     if (!dqr || !dqr.devices || dqr.devices.length === 0) {
-      log6("empty response from device service", {
+      log7("empty response from device service", {
         device: dqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 30,
         S: void 0,
         C: (f, a) => f(...a)
@@ -1707,7 +1960,7 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
       return;
     }
     invariant6(dqr, "empty response from device service", {
-      F: __dxlog_file6,
+      F: __dxlog_file7,
       L: 33,
       S: void 0,
       A: [
@@ -1717,10 +1970,10 @@ var setIdentityTags = ({ identityService, devicesService, setTag }) => {
     });
     const thisDevice = dqr.devices.find((device) => device.kind === DeviceKind.CURRENT);
     if (!thisDevice) {
-      log6("no current device", {
+      log7("no current device", {
         device: dqr
       }, {
-        F: __dxlog_file6,
+        F: __dxlog_file7,
         L: 37,
         S: void 0,
         C: (f, a) => f(...a)