cojson 0.19.20 → 0.19.22

package/src/tests/GarbageCollector.test.ts

@@ -1,7 +1,13 @@
 import { assert, beforeEach, describe, expect, test, vi } from "vitest";
 
 import { setGarbageCollectorMaxAge } from "../config";
-import { TEST_NODE_CONFIG, setupTestAccount, setupTestNode } from "./testUtils";
+import {
+  blockMessageTypeOnOutgoingPeer,
+  TEST_NODE_CONFIG,
+  setupTestAccount,
+  setupTestNode,
+} from "./testUtils";
+import { createSyncStorage } from "./testStorage.js";
 
 // We want to simulate a real world communication that happens asynchronously
 TEST_NODE_CONFIG.withAsyncPeers = true;
@@ -10,6 +16,8 @@ beforeEach(() => {
   // We want to test what happens when the garbage collector kicks in and removes a coValue
   // We set the max age to -1 to make it remove everything
   setGarbageCollectorMaxAge(-1);
+
+  setupTestNode({ isSyncServer: true });
 });
 
 describe("garbage collector", () => {
@@ -19,13 +27,14 @@ describe("garbage collector", () => {
     client.addStorage({
       ourName: "client",
     });
+    client.connectToSyncServer();
     client.node.enableGarbageCollector();
 
     const group = client.node.createGroup();
     const map = group.createMap();
     map.set("hello", "world", "trusting");
 
-    await new Promise((resolve) => setTimeout(resolve, 10));
+    await client.node.syncManager.waitForAllCoValuesSync();
 
     client.node.garbageCollector?.collect();
 
@@ -40,6 +49,7 @@ describe("garbage collector", () => {
     client.addStorage({
       ourName: "client",
     });
+    client.connectToSyncServer();
     client.node.enableGarbageCollector();
 
     const group = client.node.createGroup();
@@ -51,7 +61,7 @@ describe("garbage collector", () => {
       // This listener keeps the coValue alive
     });
 
-    await new Promise((resolve) => setTimeout(resolve, 10));
+    await client.node.syncManager.waitForAllCoValuesSync();
 
     client.node.garbageCollector?.collect();
 
@@ -66,42 +76,132 @@ describe("garbage collector", () => {
     expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
   });
 
-  test("coValues are not garbage collected if they are a group or account", async () => {
-    const client = await setupTestAccount();
+  test("coValues are not garbage collected if they are not synced with server peers", async () => {
+    const client = setupTestNode();
 
     client.addStorage({
       ourName: "client",
     });
-    client.node.enableGarbageCollector({
-      garbageCollectGroups: true,
-    });
+    client.node.enableGarbageCollector();
+    const { peer: serverPeer } = client.connectToSyncServer();
+    // Block sync with server
+    const blocker = blockMessageTypeOnOutgoingPeer(serverPeer, "content", {});
 
     const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
 
     await new Promise((resolve) => setTimeout(resolve, 10));
 
     client.node.garbageCollector?.collect();
 
-    expect(client.node.getCoValue(group.id).isAvailable()).toBe(false);
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+
+    // Resume sync with server
+    blocker.sendBlockedMessages();
+    blocker.unblock();
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // The coValue should now be collected
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+  });
+
+  test("coValues are garbage collected if there are no server peers", async () => {
+    const client = setupTestNode();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+    // Client is not connected to the sync server
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+  });
+
+  test("account coValues are not garbage collected if they have dependencies", async () => {
+    const client = await setupTestAccount({
+      // Add storage before creating the account so it's persisted
+      storage: createSyncStorage({
+        nodeName: "client",
+        storageName: "storage",
+      }),
+    });
+    // The account is created along with its profile, and the group that owns the profile
+    const profile = client.node.expectProfileLoaded(client.accountID);
+    const profileId = profile.id;
+    const profileOwnerId = profile.group.id;
+
+    client.connectToSyncServer();
+    client.node.enableGarbageCollector();
+
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // First collect removes the profile
+    client.node.garbageCollector?.collect();
+    expect(client.node.getCoValue(profileId).isAvailable()).toBe(false);
+    expect(client.node.getCoValue(profileOwnerId).isAvailable()).toBe(true);
+    expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(true);
+
+    // Second collect removes the profile owner
+    client.node.garbageCollector?.collect();
+    expect(client.node.getCoValue(profileOwnerId).isAvailable()).toBe(false);
+    expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(true);
+
+    // Third collect removes the account
+    client.node.garbageCollector?.collect();
     expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(false);
   });
 
-  test("group or account coValues are garbage collected if garbageCollectGroups is true", async () => {
-    const client = await setupTestAccount();
+  test("group coValues are garbage collected if they have no dependencies", async () => {
+    const client = setupTestNode();
 
     client.addStorage({
       ourName: "client",
     });
+    client.connectToSyncServer();
     client.node.enableGarbageCollector();
 
     const group = client.node.createGroup();
 
-    await new Promise((resolve) => setTimeout(resolve, 10));
+    await client.node.syncManager.waitForAllCoValuesSync();
 
     client.node.garbageCollector?.collect();
 
+    expect(client.node.getCoValue(group.id).isAvailable()).toBe(false);
+  });
+
+  test("group coValues are not garbage collected if they have dependencies", async () => {
+    const client = setupTestNode();
+
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await client.node.syncManager.waitForAllCoValuesSync();
+
+    // First collect removes the map
+    client.node.garbageCollector?.collect();
     expect(client.node.getCoValue(group.id).isAvailable()).toBe(true);
-    expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(true);
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+
+    // Second collect removes the group
+    client.node.garbageCollector?.collect();
+    expect(client.node.getCoValue(group.id).isAvailable()).toBe(false);
   });
 
   test("coValues are not garbage collected if the maxAge is not reached", async () => {
@@ -112,6 +212,7 @@ describe("garbage collector", () => {
     client.addStorage({
       ourName: "client",
     });
+    client.connectToSyncServer();
     client.node.enableGarbageCollector();
 
     const garbageCollector = client.node.garbageCollector;

package/src/tests/StorageApiAsync.test.ts

@@ -81,7 +81,7 @@ async function createTestNode(dbPath?: string) {
   });
 
   onTestFinished(async () => {
-    node.gracefulShutdown();
+    await node.gracefulShutdown();
     await storage.close();
   });
 
@@ -782,6 +782,55 @@ describe("StorageApiAsync", () => {
       const mapOnNode = await loadCoValueOrFail(node, map.id);
       expect(mapOnNode.get("test")).toEqual("value");
     });
+
+    test("should load dependencies again if they were unmounted", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Load the map (and its group)
+      await storage.load(map.id, callback, done);
+      callback.mockClear();
+      done.mockClear();
+
+      // Unmount the map and its group
+      storage.onCoValueUnmounted(map.id);
+      storage.onCoValueUnmounted(group.id);
+
+      // Load the map. The group dependency should be loaded again
+      await storage.load(map.id, callback, done);
+
+      expect(callback).toHaveBeenCalledTimes(2);
+      expect(callback).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          id: group.id,
+        }),
+      );
+      expect(callback).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
   });
 
   describe("waitForSync", () => {
@@ -823,4 +872,140 @@ describe("StorageApiAsync", () => {
       expect(() => storage.close()).not.toThrow();
     });
   });
+
+  describe("loadKnownState", () => {
+    test("should return cached knownState if available", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // First call should hit the database and cache the result
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(result1?.id).toBe(group.id);
+      expect(result1?.header).toBe(true);
+
+      // Second call should return from cache
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+    });
+
+    test("should return undefined for non-existent CoValue", async () => {
+      const { storage } = await createTestNode();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState("co_nonexistent" as any, resolve);
+        },
+      );
+
+      expect(result).toBeUndefined();
+    });
+
+    test("should deduplicate concurrent requests for the same ID", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Clear the cache to force database access
+      storage.knownStates.knownStates.clear();
+
+      // Spy on the database client to track how many times it's called
+      const dbClientSpy = vi.spyOn(
+        (storage as any).dbClient,
+        "getCoValueKnownState",
+      );
+
+      // Make multiple concurrent requests for the same ID
+      const promises = [
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+        new Promise<CoValueKnownState | undefined>((resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        }),
+      ];
+
+      const results = await Promise.all(promises);
+
+      // All results should be the same
+      expect(results[0]).toEqual(results[1]);
+      expect(results[1]).toEqual(results[2]);
+      expect(results[0]?.id).toBe(group.id);
+
+      // Database should only be called once due to deduplication
+      expect(dbClientSpy).toHaveBeenCalledTimes(1);
+    });
+
+    test("should use cache and not query database when cache is populated", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // Spy on the database client to track calls
+      const dbClientSpy = vi.spyOn(
+        (storage as any).dbClient,
+        "getCoValueKnownState",
+      );
+
+      // First call - should hit the database
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(dbClientSpy).toHaveBeenCalledTimes(1);
+
+      // Clear the spy to reset call count
+      dbClientSpy.mockClear();
+
+      // Second call - should use cache, not database
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+      // Database should NOT be called since cache was hit
+      expect(dbClientSpy).toHaveBeenCalledTimes(0);
+
+      // Third call - also from cache
+      const result3 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result3).toEqual(result1);
+      expect(dbClientSpy).toHaveBeenCalledTimes(0);
+    });
+  });
 });

package/src/tests/StorageApiSync.test.ts

@@ -578,6 +578,55 @@ describe("StorageApiSync", () => {
       const mapOnNode = await loadCoValueOrFail(node, map.id);
       expect(mapOnNode.get("test")).toEqual("value");
     });
+
+    test("should load dependencies again if they were unmounted", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { node, storage } = await createTestNode(dbPath);
+
+      // Create a group and a map owned by that group
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap({ test: "value" });
+      await group.core.waitForSync();
+      await map.core.waitForSync();
+
+      const callback = vi.fn((content) =>
+        node.syncManager.handleNewContent(content, "storage"),
+      );
+      const done = vi.fn();
+
+      // Load the map (and its group)
+      await storage.load(map.id, callback, done);
+      callback.mockClear();
+      done.mockClear();
+
+      // Unmount the map and its group
+      storage.onCoValueUnmounted(map.id);
+      storage.onCoValueUnmounted(group.id);
+
+      // Load the map. The group dependency should be loaded again
+      await storage.load(map.id, callback, done);
+
+      expect(callback).toHaveBeenCalledTimes(2);
+      expect(callback).toHaveBeenNthCalledWith(
+        1,
+        expect.objectContaining({
+          id: group.id,
+        }),
+      );
+      expect(callback).toHaveBeenNthCalledWith(
+        2,
+        expect.objectContaining({
+          id: map.id,
+        }),
+      );
+
+      expect(done).toHaveBeenCalledWith(true);
+
+      node.setStorage(storage);
+      const mapOnNode = await loadCoValueOrFail(node, map.id);
+      expect(mapOnNode.get("test")).toEqual("value");
+    });
   });
 
   describe("waitForSync", () => {
@@ -619,4 +668,136 @@ describe("StorageApiSync", () => {
       expect(() => storage.close()).not.toThrow();
     });
   });
+
+  describe("loadKnownState", () => {
+    test("should return correct knownState structure for existing CoValue", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(group.id);
+      expect(result?.header).toBe(true);
+      expect(result?.sessions).toEqual(group.core.knownState().sessions);
+    });
+
+    test("should return undefined for non-existent CoValue", async () => {
+      const { storage } = await createTestNode();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState("co_nonexistent" as any, resolve);
+        },
+      );
+
+      expect(result).toBeUndefined();
+    });
+
+    test("should handle CoValue with no sessions (header only)", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a CoValue with just a header (no transactions yet)
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+      await coValue.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(coValue.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(coValue.id);
+      expect(result?.header).toBe(true);
+      // The sessions should have one entry with lastIdx = 0 (just header)
+      expect(Object.keys(result?.sessions || {}).length).toBe(0);
+    });
+
+    test("should handle CoValue with multiple sessions", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { fixturesNode: fixturesNode2 } = await createFixturesNode(dbPath);
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a CoValue and have two nodes make transactions
+      const coValue = fixturesNode.createCoValue({
+        type: "comap",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...crypto.createdNowUnique(),
+      });
+
+      coValue.makeTransaction([{ key1: "value1" }], "trusting");
+      await coValue.waitForSync();
+
+      const coValueOnNode2 = await loadCoValueOrFail(
+        fixturesNode2,
+        coValue.id as CoID<RawCoMap>,
+      );
+
+      coValueOnNode2.set("key2", "value2", "trusting");
+      await coValueOnNode2.core.waitForSync();
+
+      const result = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(coValue.id, resolve);
+        },
+      );
+
+      expect(result).toBeDefined();
+      expect(result?.id).toBe(coValue.id);
+      expect(result?.header).toBe(true);
+      // Should have two sessions
+      expect(Object.keys(result?.sessions || {}).length).toBe(2);
+      // Verify sessions match the expected state
+      expect(result?.sessions).toEqual(
+        coValueOnNode2.core.knownState().sessions,
+      );
+    });
+
+    test("should use cache when knownState is cached", async () => {
+      const { fixturesNode, dbPath } = await createFixturesNode();
+      const { storage } = await createTestNode(dbPath);
+
+      // Create a group to have data in the database
+      const group = fixturesNode.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      // First call should hit the database and cache the result
+      const result1 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result1).toBeDefined();
+      expect(result1?.id).toBe(group.id);
+      expect(result1?.header).toBe(true);
+
+      // Second call should return from cache
+      const result2 = await new Promise<CoValueKnownState | undefined>(
+        (resolve) => {
+          storage.loadKnownState(group.id, resolve);
+        },
+      );
+
+      expect(result2).toEqual(result1);
+    });
+  });
 });

package/src/tests/SyncManager.processQueues.test.ts

@@ -70,7 +70,7 @@ describe("SyncManager.processQueues", () => {
       await loadCoValueOrFail(client.node, map.id);
 
       // Restart and load from storage
-      client.restart();
+      await client.restart();
       client.connectToSyncServer();
       client.addStorage({ storage });
 

package/src/tests/SyncStateManager.test.ts

@@ -46,7 +46,7 @@ describe("SyncStateManager", () => {
     const newPeerState = client.node.syncManager.peers[peerState.id]!;
 
     expect(updateSpy).toHaveBeenCalledWith(
-      peerState.id,
+      expect.objectContaining({ id: peerState.id }),
       newPeerState.getKnownState(map.core.id)!,
       { uploaded: true },
     );

package/src/tests/coPlainText.test.ts

@@ -355,7 +355,7 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
 
   await coValue.waitForSync();
 
-  client.restart();
+  await client.restart();
   client.addStorage({
     storage,
   });

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -36,6 +36,7 @@ function createMockStorage(
     ) => void;
     store?: (data: any, correctionCallback: any) => void;
     getKnownState?: (id: RawCoID) => any;
+    loadKnownState?: (id: string, callback: (knownState: any) => void) => void;
     waitForSync?: (id: string, coValue: any) => Promise<void>;
     trackCoValuesSyncState?: (
       operations: Array<{ id: RawCoID; peerId: PeerID; synced: boolean }>,
@@ -44,6 +45,7 @@ function createMockStorage(
       callback: (unsyncedCoValueIDs: RawCoID[]) => void,
     ) => void;
     stopTrackingSyncState?: (id: RawCoID) => void;
+    onCoValueUnmounted?: (id: RawCoID) => void;
     close?: () => Promise<unknown> | undefined;
   } = {},
 ): StorageAPI {
@@ -51,10 +53,13 @@ function createMockStorage(
     load: opts.load || vi.fn(),
     store: opts.store || vi.fn(),
     getKnownState: opts.getKnownState || vi.fn(),
+    loadKnownState:
+      opts.loadKnownState || vi.fn((id, callback) => callback(undefined)),
     waitForSync: opts.waitForSync || vi.fn().mockResolvedValue(undefined),
     trackCoValuesSyncState: opts.trackCoValuesSyncState || vi.fn(),
     getUnsyncedCoValueIDs: opts.getUnsyncedCoValueIDs || vi.fn(),
     stopTrackingSyncState: opts.stopTrackingSyncState || vi.fn(),
+    onCoValueUnmounted: opts.onCoValueUnmounted || vi.fn(),
     close: opts.close || vi.fn().mockResolvedValue(undefined),
   };
 }