cojson 0.19.20 → 0.19.22

package/src/sync.ts

@@ -25,6 +25,7 @@ import {
   CoValueKnownState,
   knownStateFrom,
   KnownStateSessions,
+  peerHasAllContent,
 } from "./knownState.js";
 import { StorageAPI } from "./storage/index.js";
 
@@ -101,6 +102,10 @@ export interface Peer {
   persistent?: boolean;
 }
 
+function isPersistentServerPeer(peer: Peer | PeerState): boolean {
+  return peer.role === "server" && (peer.persistent ?? false);
+}
+
 export type ServerPeerSelector = (
   id: RawCoID,
   serverPeers: PeerState[],
@@ -353,7 +358,7 @@ export class SyncManager {
   }
 
   startPeerReconciliation(peer: PeerState) {
-    if (peer.role === "server" && peer.persistent) {
+    if (isPersistentServerPeer(peer)) {
       // Resume syncing unsynced CoValues asynchronously
       this.resumeUnsyncedCoValues().catch((error) => {
         logger.warn("Failed to resume unsynced CoValues:", error);
@@ -524,7 +529,7 @@ export class SyncManager {
 
     const unsubscribeFromKnownStatesUpdates =
       peerState.subscribeToKnownStatesUpdates((id, knownState) => {
-        this.syncState.triggerUpdate(peer.id, id, knownState.value());
+        this.syncState.triggerUpdate(peer, id, knownState.value());
       });
 
     if (!skipReconciliation && peerState.role === "server") {
@@ -583,33 +588,125 @@ export class SyncManager {
     peer.setKnownState(msg.id, knownStateFrom(msg));
     const coValue = this.local.getCoValue(msg.id);
 
+    // Fast path: CoValue is already in memory
     if (coValue.isAvailable()) {
       this.sendNewContent(msg.id, peer);
       return;
     }
 
-    const peers = this.getServerPeers(msg.id, peer.id);
+    const peerKnownState = peer.getOptimisticKnownState(msg.id);
+
+    // Fast path: Peer has no content at all - skip lazy load check, just load directly
+    if (!peerKnownState?.header) {
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+      return;
+    }
+
+    // Check storage knownState before doing full load (lazy load optimization)
+    coValue.getKnownStateFromStorage((storageKnownState) => {
+      // Race condition: CoValue might have been loaded while we were waiting for storage
+      if (coValue.isAvailable()) {
+        this.sendNewContent(msg.id, peer);
+        return;
+      }
+
+      if (!storageKnownState) {
+        // Not in storage, try loading from peers
+        this.loadFromPeersAndRespond(msg.id, peer, coValue);
+        return;
+      }
+
+      // Check if peer already has all content
+      if (peerHasAllContent(storageKnownState, peerKnownState)) {
+        // Peer already has everything - reply with known message, no full load needed
+        peer.trackToldKnownState(msg.id);
+        this.trySendToPeer(peer, {
+          action: "known",
+          ...storageKnownState,
+        });
+        return;
+      }
 
-    coValue.load(peers);
+      // Peer needs content - do full load from storage
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+    });
+  }
+
+  /**
+   * Helper to load from storage and respond appropriately.
+   * Falls back to peers if not found in storage.
+   */
+  private loadFromStorageAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    coValue.loadFromStorage((found) => {
+      if (found && coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
+      } else {
+        this.loadFromPeersAndRespond(id, peer, coValue);
+      }
+    });
+  }
+
+  /**
+   * Helper to load from peers and respond appropriately.
+   */
+  private loadFromPeersAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    const peers = this.getServerPeers(id, peer.id);
+    coValue.loadFromPeers(peers);
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
         return;
       }
+      this.handleLoadNotFound(id, peer);
+    };
 
-      peer.trackToldKnownState(msg.id);
+    if (peers.length > 0) {
+      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
+    } else {
+      handleLoadResult();
+    }
+  }
+
+  /**
+   * Handle case when CoValue is not found.
+   */
+  private handleLoadNotFound(id: RawCoID, peer: PeerState) {
+    peer.trackToldKnownState(id);
+    this.trySendToPeer(peer, {
+      action: "known",
+      id,
+      header: false,
+      sessions: {},
+    });
+  }
+
+  /**
+   * Request full content from a peer when we don't have the CoValue.
+   */
+  private requestFullContent(id: RawCoID, peer: PeerState | undefined) {
+    if (peer) {
       this.trySendToPeer(peer, {
         action: "known",
-        id: msg.id,
+        isCorrection: true,
+        id,
         header: false,
         sessions: {},
       });
-    };
-
-    if (peers.length > 0 || this.local.storage) {
-      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
     } else {
-      handleLoadResult();
+      // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
+      // Should never happen
+      logger.error("Received new content with no header on a missing CoValue", {
+        id,
+      });
     }
   }
 
@@ -618,8 +715,8 @@ export class SyncManager {
 
     peer.combineWith(msg.id, knownStateFrom(msg));
 
-    // The header is a boolean value that tells us if the other peer do have information about the header.
-    // If it's false in this point it means that the coValue is unavailable on the other peer.
+    // The header is a boolean value that tells us if the other peer has information about the header.
+    // If it's false at this point it means that the coValue is unavailable on the other peer.
     const availableOnPeer = peer.getOptimisticKnownState(msg.id)?.header;
 
     if (!availableOnPeer) {
@@ -694,46 +791,37 @@ export class SyncManager {
      */
     if (!coValue.hasVerifiedContent()) {
       /**
-       * The peer has assumed we already have the CoValue
+       * The peer/import has assumed we already have the CoValue
        */
       if (!msg.header) {
-        // We check if the covalue was in memory and has been garbage collected
-        // In that case we should have it tracked in the storage
-        const storageKnownState = this.local.storage?.getKnownState(msg.id);
-
-        if (storageKnownState?.header) {
-          // If the CoValue has been garbage collected, we load it from the storage before handling the new content
-          coValue.loadFromStorage((found) => {
-            if (found) {
-              this.handleNewContent(msg, from);
-            } else {
-              logger.error("Known CoValue not found in storage", {
-                id: msg.id,
-              });
-            }
-          });
-          return;
-        }
-
-        // The peer assumption is not correct, so we ask for the full CoValue
-        if (peer) {
-          this.trySendToPeer(peer, {
-            action: "known",
-            isCorrection: true,
-            id: msg.id,
-            header: false,
-            sessions: {},
-          });
-        } else {
-          // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
-          // Should never happen
-          logger.error(
-            "Received new content with no header on a missing CoValue",
+        // Content from storage without header - this can happen if:
+        // 1. Storage is streaming a large CoValue in chunks
+        // 2. Server is under heavy load, so a chunk isn't processed for a long time
+        // 3. GC cleanup unmounts the CoValue while streaming is in progress
+        // 4. The chunk is finally processed, but the CoValue is no longer available
+        // TODO: Fix this by either not unmounting CoValues with active streaming,
+        // or by cleaning up the streaming queue on unmount
+        if (from === "storage") {
+          logger.warn(
+            "Received content from storage without header - CoValue may have been garbage collected mid-stream",
             {
               id: msg.id,
+              from,
             },
           );
+          return;
         }
+
+        // Try to load from storage - the CoValue might have been garbage collected from memory
+        coValue.loadFromStorage((found) => {
+          if (found) {
+            // CoValue was in storage, process the new content
+            this.handleNewContent(msg, from);
+          } else {
+            // CoValue not in storage, ask peer for full content
+            this.requestFullContent(msg.id, peer);
+          }
+        });
         return;
       }
 
@@ -992,6 +1080,18 @@ export class SyncManager {
     const isSyncRequired = this.local.syncWhen !== "never";
     if (isSyncRequired && peers.length === 0) {
       this.unsyncedTracker.add(coValueId);
+
+      // Mark CoValue as synced once a persistent server peer is added and
+      // the CoValue is synced
+      const unsubscribe = this.syncState.subscribeToCoValueUpdates(
+        coValueId,
+        (peer, _knownState, syncState) => {
+          if (isPersistentServerPeer(peer) && syncState.uploaded) {
+            this.unsyncedTracker.remove(coValueId);
+            unsubscribe();
+          }
+        },
+      );
       return;
     }
 
@@ -1044,6 +1144,19 @@ export class SyncManager {
     });
   }
 
+  /**
+   * Returns true if the local CoValue changes have been synced to all persistent server peers.
+   *
+   * Used during garbage collection to determine if the coValue is pending sync.
+   */
+  isSyncedToServerPeers(id: RawCoID): boolean {
+    // If there are currently no server peers, go ahead with GC.
+    // The CoValue will be reloaded into memory and synced when a peer is added.
+    return this.getPersistentServerPeers(id).every((peer) =>
+      this.syncState.isSynced(peer, id),
+    );
+  }
+
   waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const peerState = this.peers[peerId];
 

package/src/tests/CojsonMessageChannel.test.ts

@@ -0,0 +1,306 @@
+import { describe, test, expect, beforeEach, assert } from "vitest";
+import { CojsonMessageChannel } from "../CojsonMessageChannel";
+import type { Peer } from "../sync.js";
+import type { RawCoMap } from "../coValues/coMap.js";
+import {
+  setupTestNode,
+  SyncMessagesLog,
+  waitFor,
+  createTrackedMessageChannel,
+  createMockWorkerWithAccept,
+  loadCoValueOrFail,
+} from "./testUtils";
+
+describe("CojsonMessageChannel", () => {
+  beforeEach(() => {
+    SyncMessagesLog.clear();
+  });
+
+  test("should sync data between two contexts via MessageChannel", async () => {
+    // Create two nodes using setupTestNode (handles cleanup automatically)
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      // This runs in the "worker" context
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    // Host side: expose to the mock worker
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+      messageChannel: createTrackedMessageChannel({
+        port1Name: "client",
+        port2Name: "server",
+      }),
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1
+    const group = node1.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("key", "value", "trusting");
+
+    // Verify data synced
+    const mapOnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map.id);
+    expect(mapOnNode2.get("key")).toBe("value");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Map: map.core,
+        Group: group.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "server -> client | LOAD Map sessions: empty",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> client | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("should handle disconnection correctly", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const peerId = "disconnect-test-peer";
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        id: peerId,
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      id: peerId,
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify peers are connected (same ID on both sides)
+    expect(node1.syncManager.peers["disconnect-test-peer"]).toBeDefined();
+    expect(node2.syncManager.peers["disconnect-test-peer"]).toBeDefined();
+
+    peer1.outgoing.close();
+
+    expect(node1.syncManager.peers["disconnect-test-peer"]).toBeUndefined();
+
+    await waitFor(() => {
+      expect(node2.syncManager.peers["disconnect-test-peer"]).toBeUndefined();
+    });
+  });
+
+  test("should ignore mismatched peer IDs in waitForConnection() when id filter is provided", async () => {
+    const { node } = setupTestNode();
+
+    const hostPeerId = "host-peer-id";
+    const wrongPeerId = "wrong-peer-id";
+
+    let acceptPromiseResolved = false;
+
+    // Mock worker that expects a different ID
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      // This should not resolve because the ID doesn't match
+      const acceptPromise = CojsonMessageChannel.acceptFromPort(port, {
+        id: wrongPeerId, // Expecting a different ID
+        role: "server",
+      });
+
+      // Set a timeout to detect if it's waiting
+      const timeoutPromise = new Promise<null>((resolve) =>
+        setTimeout(() => resolve(null), 100),
+      );
+
+      const result = await Promise.race([acceptPromise, timeoutPromise]);
+      if (result !== null) {
+        acceptPromiseResolved = true;
+        node.syncManager.addPeer(result);
+      }
+    });
+
+    // Expose with a different ID than what accept expects
+    CojsonMessageChannel.expose(mockWorker, {
+      id: hostPeerId,
+      role: "client",
+    });
+
+    // Wait a bit to ensure the accept didn't resolve
+    await new Promise((resolve) => setTimeout(resolve, 150));
+
+    // The accept should not have resolved because IDs don't match
+    expect(acceptPromiseResolved).toBe(false);
+  });
+
+  test("should sync data bidirectionally", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1
+    const group1 = node1.createGroup();
+    group1.addMember("everyone", "writer");
+    const map1 = group1.createMap();
+    map1.set("from", "node1", "trusting");
+
+    // Create data on node2
+    const group2 = node2.createGroup();
+    group2.addMember("everyone", "writer");
+    const map2 = group2.createMap();
+    map2.set("from", "node2", "trusting");
+
+    // Verify data synced in both directions
+    const map1OnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map1.id);
+    expect(map1OnNode2.get("from")).toBe("node1");
+
+    const map2OnNode1 = await loadCoValueOrFail<RawCoMap>(node1, map2.id);
+    expect(map2OnNode1.get("from")).toBe("node2");
+  });
+
+  test("should invoke onClose callback when connection closes", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let onCloseCalledOnHost = false;
+    let onCloseCalledOnWorker = false;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+        onClose: () => {
+          onCloseCalledOnWorker = true;
+        },
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+      onClose: () => {
+        onCloseCalledOnHost = true;
+      },
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Close the connection
+    peer1.outgoing.close();
+
+    // Wait for close to propagate
+    await waitFor(() => {
+      expect(onCloseCalledOnHost).toBe(true);
+    });
+
+    await waitFor(() => {
+      expect(onCloseCalledOnWorker).toBe(true);
+    });
+  });
+
+  test("should apply role configuration correctly", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify roles are correctly set
+    expect(peer1.role).toBe("client");
+    expect(peer2).not.toBeNull();
+    expect(peer2!.role).toBe("server");
+  });
+
+  test("should generate and use the same peer ID on both sides when not provided", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    // Don't provide an id - it should be auto-generated
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify peer1 has an auto-generated ID
+    expect(peer1.id).toMatch(/^channel_/);
+
+    // Verify both peers have the same ID
+    expect(peer2).not.toBeNull();
+    expect(peer2!.id).toBe(peer1.id);
+
+    // Verify the peer is accessible in both sync managers with the same ID
+    expect(node1.syncManager.peers[peer1.id]).toBeDefined();
+    expect(node2.syncManager.peers[peer1.id]).toBeDefined();
+  });
+
+  test("should handle delayed addPeer on accept side", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const delay = new Promise((resolve) => setTimeout(resolve, 50));
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      // Deliberately delay adding the peer
+      await delay;
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1 immediately (before node2 has added the peer)
+    const group = node1.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("key", "value", "trusting");
+
+    await delay;
+
+    // Verify data synced despite the delay
+    const mapOnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map.id);
+    expect(mapOnNode2.get("key")).toBe("value");
+  });
+});