cojson 0.19.20 → 0.19.21

package/src/sync.ts

@@ -25,6 +25,7 @@ import {
   CoValueKnownState,
   knownStateFrom,
   KnownStateSessions,
+  peerHasAllContent,
 } from "./knownState.js";
 import { StorageAPI } from "./storage/index.js";
 
@@ -583,33 +584,125 @@ export class SyncManager {
     peer.setKnownState(msg.id, knownStateFrom(msg));
     const coValue = this.local.getCoValue(msg.id);
 
+    // Fast path: CoValue is already in memory
     if (coValue.isAvailable()) {
       this.sendNewContent(msg.id, peer);
       return;
     }
 
-    const peers = this.getServerPeers(msg.id, peer.id);
+    const peerKnownState = peer.getOptimisticKnownState(msg.id);
 
-    coValue.load(peers);
+    // Fast path: Peer has no content at all - skip lazy load check, just load directly
+    if (!peerKnownState?.header) {
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+      return;
+    }
+
+    // Check storage knownState before doing full load (lazy load optimization)
+    coValue.getKnownStateFromStorage((storageKnownState) => {
+      // Race condition: CoValue might have been loaded while we were waiting for storage
+      if (coValue.isAvailable()) {
+        this.sendNewContent(msg.id, peer);
+        return;
+      }
+
+      if (!storageKnownState) {
+        // Not in storage, try loading from peers
+        this.loadFromPeersAndRespond(msg.id, peer, coValue);
+        return;
+      }
+
+      // Check if peer already has all content
+      if (peerHasAllContent(storageKnownState, peerKnownState)) {
+        // Peer already has everything - reply with known message, no full load needed
+        peer.trackToldKnownState(msg.id);
+        this.trySendToPeer(peer, {
+          action: "known",
+          ...storageKnownState,
+        });
+        return;
+      }
+
+      // Peer needs content - do full load from storage
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+    });
+  }
+
+  /**
+   * Helper to load from storage and respond appropriately.
+   * Falls back to peers if not found in storage.
+   */
+  private loadFromStorageAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    coValue.loadFromStorage((found) => {
+      if (found && coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
+      } else {
+        this.loadFromPeersAndRespond(id, peer, coValue);
+      }
+    });
+  }
+
+  /**
+   * Helper to load from peers and respond appropriately.
+   */
+  private loadFromPeersAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    const peers = this.getServerPeers(id, peer.id);
+    coValue.loadFromPeers(peers);
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
         return;
       }
+      this.handleLoadNotFound(id, peer);
+    };
 
-      peer.trackToldKnownState(msg.id);
+    if (peers.length > 0) {
+      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
+    } else {
+      handleLoadResult();
+    }
+  }
+
+  /**
+   * Handle case when CoValue is not found.
+   */
+  private handleLoadNotFound(id: RawCoID, peer: PeerState) {
+    peer.trackToldKnownState(id);
+    this.trySendToPeer(peer, {
+      action: "known",
+      id,
+      header: false,
+      sessions: {},
+    });
+  }
+
+  /**
+   * Request full content from a peer when we don't have the CoValue.
+   */
+  private requestFullContent(id: RawCoID, peer: PeerState | undefined) {
+    if (peer) {
       this.trySendToPeer(peer, {
         action: "known",
-        id: msg.id,
+        isCorrection: true,
+        id,
         header: false,
         sessions: {},
       });
-    };
-
-    if (peers.length > 0 || this.local.storage) {
-      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
     } else {
-      handleLoadResult();
+      // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
+      // Should never happen
+      logger.error("Received new content with no header on a missing CoValue", {
+        id,
+      });
     }
   }
 
@@ -694,46 +787,37 @@ export class SyncManager {
      */
     if (!coValue.hasVerifiedContent()) {
       /**
-       * The peer has assumed we already have the CoValue
+       * The peer/import has assumed we already have the CoValue
        */
       if (!msg.header) {
-        // We check if the covalue was in memory and has been garbage collected
-        // In that case we should have it tracked in the storage
-        const storageKnownState = this.local.storage?.getKnownState(msg.id);
-
-        if (storageKnownState?.header) {
-          // If the CoValue has been garbage collected, we load it from the storage before handling the new content
-          coValue.loadFromStorage((found) => {
-            if (found) {
-              this.handleNewContent(msg, from);
-            } else {
-              logger.error("Known CoValue not found in storage", {
-                id: msg.id,
-              });
-            }
-          });
-          return;
-        }
-
-        // The peer assumption is not correct, so we ask for the full CoValue
-        if (peer) {
-          this.trySendToPeer(peer, {
-            action: "known",
-            isCorrection: true,
-            id: msg.id,
-            header: false,
-            sessions: {},
-          });
-        } else {
-          // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
-          // Should never happen
-          logger.error(
-            "Received new content with no header on a missing CoValue",
+        // Content from storage without header - this can happen if:
+        // 1. Storage is streaming a large CoValue in chunks
+        // 2. Server is under heavy load, so a chunk isn't processed for a long time
+        // 3. GC cleanup unmounts the CoValue while streaming is in progress
+        // 4. The chunk is finally processed, but the CoValue is no longer available
+        // TODO: Fix this by either not unmounting CoValues with active streaming,
+        // or by cleaning up the streaming queue on unmount
+        if (from === "storage") {
+          logger.warn(
+            "Received content from storage without header - CoValue may have been garbage collected mid-stream",
             {
               id: msg.id,
+              from,
             },
           );
+          return;
         }
+
+        // Try to load from storage - the CoValue might have been garbage collected from memory
+        coValue.loadFromStorage((found) => {
+          if (found) {
+            // CoValue was in storage, process the new content
+            this.handleNewContent(msg, from);
+          } else {
+            // CoValue not in storage, ask peer for full content
+            this.requestFullContent(msg.id, peer);
+          }
+        });
         return;
       }
 

package/src/tests/StorageApiAsync.test.ts

@@ -823,4 +823,140 @@ describe("StorageApiAsync", () => {
       expect(() => storage.close()).not.toThrow();
     });
   });
+
+  describe("loadKnownState", () => {
+    test("should return cached knownState if available", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // First call should hit the database and cache the result
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(result1?.id).toBe(group.id);
+      expect(result1?.header).toBe(true);
+
+      // Second call should return from cache
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+    });
+
+    test("should return undefined for non-existent CoValue", async () => {
+      const { storage } = await createTestNode();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState("co_nonexistent" as any, resolve);
+        },
+      );
+
+      expect(result).toBeUndefined();
+    });
+
+    test("should deduplicate concurrent requests for the same ID", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Clear the cache to force database access
+      storage.knownStates.knownStates.clear();
+
+      // Spy on the database client to track how many times it's called
+      const dbClientSpy = vi.spyOn(
+        (storage as any).dbClient,
+        "getCoValueKnownState",
+      );
+
+      // Make multiple concurrent requests for the same ID
+      const promises = [
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+      ];
+
+      const results = await Promise.all(promises);
+
+      // All results should be the same
+      expect(results[0]).toEqual(results[1]);
+      expect(results[1]).toEqual(results[2]);
+      expect(results[0]?.id).toBe(group.id);
+
+      // Database should only be called once due to deduplication
+      expect(dbClientSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should use cache and not query database when cache is populated", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Spy on the database client to track calls
+      const dbClientSpy = vi.spyOn(
+        (storage as any).dbClient,
+        "getCoValueKnownState",
+      );
+
+      // First call - should hit the database
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(dbClientSpy).toHaveBeenCalledTimes(1);
+
+      // Clear the spy to reset call count
+      dbClientSpy.mockClear();
+
+      // Second call - should use cache, not database
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+      // Database should NOT be called since cache was hit
+      expect(dbClientSpy).toHaveBeenCalledTimes(0);
+
+      // Third call - also from cache
+      const result3 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result3).toEqual(result1);
+      expect(dbClientSpy).toHaveBeenCalledTimes(0);
+    });
+  });
 });

package/src/tests/StorageApiSync.test.ts

@@ -619,4 +619,136 @@ describe("StorageApiSync", () => {
       expect(() => storage.close()).not.toThrow();
     });
   });
+
+  describe("loadKnownState", () => {
+    test("should return correct knownState structure for existing CoValue", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(group.id);
+      expect(result?.header).toBe(true);
+      expect(result?.sessions).toEqual(group.core.knownState().sessions);
+    });
+
+    test("should return undefined for non-existent CoValue", async () => {
+      const { storage } = await createTestNode();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState("co_nonexistent" as any, resolve);
+        },
+      );
+
+      expect(result).toBeUndefined();
+    });
+
+    test("should handle CoValue with no sessions (header only)", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a CoValue with just a header (no transactions yet)
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+      await coValue.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(coValue.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(coValue.id);
+      expect(result?.header).toBe(true);
+      // The sessions should have one entry with lastIdx = 0 (just header)
+      expect(Object.keys(result?.sessions || {}).length).toBe(0);
+    });
+
+    test("should handle CoValue with multiple sessions", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { fixturesNode: fixturesNode2 } = await createFixturesNode(dbPath);
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a CoValue and have two nodes make transactions
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      coValue.makeTransaction([{ key1: "value1" }], "trusting");
+      await coValue.waitForSync();
+
+      const coValueOnNode2 = await loadCoValueOrFail(
+        fixturesNode2,
+        coValue.id as CoID<RawCoMap>,
+      );
+
+      coValueOnNode2.set("key2", "value2", "trusting");
+      await coValueOnNode2.core.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(coValue.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(coValue.id);
+      expect(result?.header).toBe(true);
+      // Should have two sessions
+      expect(Object.keys(result?.sessions || {}).length).toBe(2);
+      // Verify sessions match the expected state
+      expect(result?.sessions).toEqual(
+        coValueOnNode2.core.knownState().sessions,
+      );
+    });
+
+    test("should use cache when knownState is cached", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // First call should hit the database and cache the result
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(result1?.id).toBe(group.id);
+      expect(result1?.header).toBe(true);
+
+      // Second call should return from cache
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+    });
+  });
 });

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -36,6 +36,7 @@ function createMockStorage(
     ) => void;
     store?: (data: any, correctionCallback: any) => void;
     getKnownState?: (id: RawCoID) => any;
+    loadKnownState?: (id: string, callback: (knownState: any) => void) => void;
     waitForSync?: (id: string, coValue: any) => Promise<void>;
     trackCoValuesSyncState?: (
       operations: Array<{ id: RawCoID; peerId: PeerID; synced: boolean }>,
@@ -51,6 +52,8 @@ function createMockStorage(
     load: opts.load || vi.fn(),
     store: opts.store || vi.fn(),
     getKnownState: opts.getKnownState || vi.fn(),
+    loadKnownState:
+      opts.loadKnownState || vi.fn((id, callback) => callback(undefined)),
     waitForSync: opts.waitForSync || vi.fn().mockResolvedValue(undefined),
     trackCoValuesSyncState: opts.trackCoValuesSyncState || vi.fn(),
     getUnsyncedCoValueIDs: opts.getUnsyncedCoValueIDs || vi.fn(),