cojson 0.16.3 → 0.16.5

package/src/tests/sync.mesh.test.ts

@@ -142,19 +142,26 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "edge-france -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "edge-france -> core | CONTENT ParentGroup header: true new: After: 0 New: 6",
-        "edge-france -> core | CONTENT Group header: true new: After: 0 New: 5",
+        "edge-france -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "edge-france -> core | CONTENT Group header: true new: After: 0 New: 3",
         "edge-france -> storage | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "edge-france -> core | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "edge-france -> storage | CONTENT Group header: false new: After: 3 New: 2",
+        "edge-france -> core | CONTENT Group header: false new: After: 3 New: 2",
         "edge-france -> storage | CONTENT Map header: true new: After: 0 New: 1",
         "edge-france -> core | CONTENT Map header: true new: After: 0 New: 1",
+        "core -> edge-france | KNOWN Group sessions: header/3",
+        "core -> storage | CONTENT Group header: true new: After: 0 New: 3",
         "core -> edge-france | KNOWN ParentGroup sessions: header/6",
         "core -> storage | CONTENT ParentGroup header: true new: After: 0 New: 6",
         "core -> edge-france | KNOWN Group sessions: header/5",
-        "core -> storage | CONTENT Group header: true new: After: 0 New: 5",
+        "core -> storage | CONTENT Group header: false new: After: 3 New: 2",
         "core -> edge-france | KNOWN Map sessions: header/1",
         "core -> storage | CONTENT Map header: true new: After: 0 New: 1",
+        "edge-france -> core | CONTENT ParentGroup header: true new: ",
         "client -> edge-italy | LOAD Map sessions: empty",
+        "core -> edge-france | KNOWN ParentGroup sessions: header/6",
+        "core -> storage | CONTENT ParentGroup header: true new: ",
         "edge-italy -> storage | LOAD Map sessions: empty",
         "storage -> edge-italy | KNOWN Map sessions: empty",
         "edge-italy -> core | LOAD Map sessions: empty",
@@ -509,8 +516,7 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
     ).toMatchInlineSnapshot(`
       [
         "edge -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "edge -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "edge -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "edge -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "edge -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "edge -> storage | CONTENT Map header: false new: After: 146 New: 54",
       ]

package/src/tests/sync.peerReconciliation.test.ts

@@ -47,6 +47,8 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN Map sessions: empty",
         "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN Map sessions: header/1",
+        "client -> server | CONTENT Group header: true new: ",
+        "client -> server | CONTENT Map header: true new: ",
       ]
     `);
   });
@@ -203,7 +205,7 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN CORRECTION Map sessions: empty",
         "client -> server | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | LOAD Group sessions: empty",
-        "server -> client | KNOWN Map sessions: empty",
+        "server -> client | KNOWN Map sessions: header/2",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "server -> client | KNOWN Group sessions: header/3",
       ]
@@ -276,8 +278,8 @@ describe("peer reconciliation", () => {
         "client -> server | CONTENT Profile header: true new: After: 0 New: 1",
         "client -> server | CONTENT Map header: false new: After: 1 New: 1",
         "server -> client | LOAD Account sessions: empty",
-        "server -> client | KNOWN ProfileGroup sessions: empty",
-        "server -> client | KNOWN Profile sessions: empty",
+        "server -> client | KNOWN ProfileGroup sessions: header/0",
+        "server -> client | KNOWN Profile sessions: header/0",
         "server -> client | KNOWN CORRECTION Map sessions: empty",
         "client -> server | CONTENT Account header: true new: After: 0 New: 4",
         "client -> server | CONTENT Map header: true new: After: 0 New: 2",
@@ -285,7 +287,7 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN ProfileGroup sessions: header/5",
         "server -> client | KNOWN Profile sessions: header/1",
         "server -> client | LOAD Group sessions: empty",
-        "server -> client | KNOWN Map sessions: empty",
+        "server -> client | KNOWN Map sessions: header/2",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "server -> client | KNOWN Group sessions: header/3",
       ]

package/src/tests/sync.storage.test.ts

@@ -181,13 +181,11 @@ describe("client with storage syncs with server", () => {
       [
         "client -> server | LOAD Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/1",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/3",
-        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
-        "client -> storage | CONTENT Map header: true new: After: 0 New: 2",
+        "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
         "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
       ]
@@ -291,20 +289,16 @@ describe("client syncs with a server with storage", () => {
       [
         "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "client -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 73",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 73",
         "client -> storage | CONTENT Map header: false new: After: 73 New: 73",
-        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
-        "client -> server | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
         "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
         "client -> server | CONTENT Map header: false new: After: 146 New: 54",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Map sessions: header/0",
-        "server -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
         "server -> client | KNOWN Map sessions: header/73",
-        "server -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "server -> client | KNOWN Map sessions: header/146",
         "server -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "server -> client | KNOWN Map sessions: header/200",
@@ -410,7 +404,7 @@ describe("client syncs with a server with storage", () => {
 
     const correctionSpy = vi.fn();
 
-    client.node.storage?.store(newContentChunks.slice(1, 2), correctionSpy);
+    client.node.storage?.store(newContentChunks[1]!, correctionSpy);
 
     expect(correctionSpy).not.toHaveBeenCalled();
 

package/src/tests/sync.storageAsync.test.ts

@@ -16,6 +16,7 @@ describe("client with storage syncs with server", () => {
   let jazzCloud: ReturnType<typeof setupTestNode>;
 
   beforeEach(async () => {
+    vi.resetAllMocks();
     SyncMessagesLog.clear();
     jazzCloud = setupTestNode({
       isSyncServer: true,
@@ -174,18 +175,46 @@ describe("client with storage syncs with server", () => {
       [
         "client -> server | LOAD Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/1",
-        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
         "server -> client | CONTENT Map header: false new: After: 1 New: 1",
-        "client -> server | KNOWN Group sessions: header/3",
-        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
-        "client -> storage | CONTENT Map header: true new: After: 0 New: 2",
+        "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
         "client -> server | KNOWN Map sessions: header/2",
         "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
       ]
     `);
   });
+
+  test("the order of updates between CoValues should be preserved to ensure consistency in case of shutdown in the middle of sync", async () => {
+    const client = setupTestNode();
+
+    await client.addAsyncStorage();
+
+    const group = client.node.createGroup();
+    const initialMap = group.createMap();
+
+    const child = group.createMap();
+    child.set("parent", initialMap.id);
+    initialMap.set("child", child.id);
+
+    await initialMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        InitialMap: initialMap.core,
+        ChildMap: child.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> storage | CONTENT InitialMap header: true new: ",
+        "client -> storage | CONTENT ChildMap header: true new: After: 0 New: 1",
+        "client -> storage | CONTENT InitialMap header: false new: After: 0 New: 1",
+      ]
+    `);
+  });
 });
 
 describe("client syncs with a server with storage", () => {
@@ -271,20 +300,16 @@ describe("client syncs with a server with storage", () => {
       [
         "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "client -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 73",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 73",
         "client -> storage | CONTENT Map header: false new: After: 73 New: 73",
-        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
-        "client -> server | CONTENT Map header: true new:  expectContentUntil: header/200",
-        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
         "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
         "client -> server | CONTENT Map header: false new: After: 146 New: 54",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> storage | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Map sessions: header/0",
-        "server -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
         "server -> client | KNOWN Map sessions: header/73",
-        "server -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> storage | CONTENT Map header: true new: After: 0 New: 73",
         "server -> client | KNOWN Map sessions: header/146",
         "server -> storage | CONTENT Map header: false new: After: 73 New: 73",
         "server -> client | KNOWN Map sessions: header/200",
@@ -369,7 +394,7 @@ describe("client syncs with a server with storage", () => {
 
     const correctionSpy = vi.fn();
 
-    client.node.storage?.store(newContentChunks.slice(1, 2), correctionSpy);
+    client.node.storage?.store(newContentChunks[1]!, correctionSpy);
 
     // Wait for the content to be stored in the storage
     // We can't use waitForSync because we are trying to store stale data

package/src/tests/sync.test.ts

@@ -100,15 +100,18 @@ test("Can sync a coValue with private transactions through a server to another c
 
   const map = group.createMap();
   map.set("hello", "world", "private");
+
   group.addMember("everyone", "reader");
 
   const { node: client2 } = await setupTestAccount({
     connected: true,
   });
 
-  const mapOnClient2 = await loadCoValueOrFail(client2, map.id);
+  await waitFor(async () => {
+    const loadedMap = await loadCoValueOrFail(client2, map.id);
 
-  expect(mapOnClient2.get("hello")).toEqual("world");
+    expect(loadedMap.get("hello")).toEqual("world");
+  });
 });
 
 test("should keep the peer state when the peer closes if persistent is true", async () => {
@@ -563,8 +566,6 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
     const mapOnClient = group.createMap();
     mapOnClient.set("key1", "value1", "trusting");
 
-    await client.syncManager.syncCoValue(mapOnClient.core);
-
     // Wait for the full sync to complete
     await mapOnClient.core.waitForSync();
 
@@ -594,7 +595,6 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
     await map.core.waitForSync();
 
     // Block the content messages
@@ -606,7 +606,7 @@ describe("SyncManager - knownStates vs optimisticKnownStates", () => {
 
     map.set("key2", "value2", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
+    await new Promise<void>(queueMicrotask);
 
     expect(peerState.optimisticKnownStates.get(map.core.id)).not.toEqual(
       peerState.knownStates.get(map.core.id),
@@ -638,8 +638,6 @@ describe("SyncManager.addPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     // Wait for initial sync
     await map.core.waitForSync();
 
@@ -671,8 +669,6 @@ describe("SyncManager.addPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     // Wait for initial sync
     await map.core.waitForSync();
 
@@ -843,8 +839,6 @@ describe("waitForSyncWithPeer", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     await expect(
       client.node.syncManager.waitForSyncWithPeer(
         peerState.id,
@@ -868,8 +862,6 @@ describe("waitForSyncWithPeer", () => {
       return Promise.resolve();
     });
 
-    await client.node.syncManager.syncCoValue(map.core);
-
     await expect(
       client.node.syncManager.waitForSyncWithPeer(
         peerState.id,

package/src/tests/sync.upload.test.ts

@@ -78,12 +78,15 @@ describe("client to server upload", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
         "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 6",
-        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Group header: false new: After: 3 New: 2",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN ParentGroup sessions: header/6",
         "server -> client | KNOWN Group sessions: header/5",
         "server -> client | KNOWN Map sessions: header/1",
+        "client -> server | CONTENT ParentGroup header: true new: ",
       ]
     `);
   });
@@ -253,6 +256,40 @@ describe("client to server upload", () => {
     `);
   });
 
+  test("local updates batching", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const initialMap = group.createMap();
+
+    const child = group.createMap();
+    child.set("parent", initialMap.id);
+    initialMap.set("child", child.id);
+
+    await initialMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        InitialMap: initialMap.core,
+        ChildMap: child.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | CONTENT InitialMap header: true new: ",
+        "client -> server | CONTENT ChildMap header: true new: After: 0 New: 1",
+        "client -> server | CONTENT InitialMap header: false new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | KNOWN InitialMap sessions: header/0",
+        "server -> client | KNOWN ChildMap sessions: header/1",
+        "server -> client | KNOWN InitialMap sessions: header/1",
+      ]
+    `);
+  });
+
   test("large coValue upload streaming", async () => {
     const client = setupTestNode({
       connected: true,

package/src/tests/testStorage.ts

@@ -5,11 +5,7 @@ import { join } from "node:path";
 import Database, { type Database as DatabaseT } from "libsql";
 import { onTestFinished } from "vitest";
 import { RawCoID, StorageAPI } from "../exports";
-import {
-  SQLiteDatabaseDriver,
-  StorageApiAsync,
-  StorageApiSync,
-} from "../storage";
+import { SQLiteDatabaseDriver } from "../storage";
 import { getSqliteStorage } from "../storage/sqlite";
 import {
   SQLiteDatabaseDriverAsync,
@@ -148,13 +144,11 @@ function trackStorageMessages(
   const originalLoad = storage.load;
 
   storage.store = function (data, correctionCallback) {
-    for (const msg of data ?? []) {
-      SyncMessagesLog.add({
-        from: nodeName,
-        to: storageName,
-        msg,
-      });
-    }
+    SyncMessagesLog.add({
+      from: nodeName,
+      to: storageName,
+      msg: data,
+    });
 
     return originalStore.call(storage, data, (correction) => {
       SyncMessagesLog.add({
@@ -167,7 +161,19 @@ function trackStorageMessages(
         },
       });
 
-      return correctionCallback(correction);
+      const correctionMessages = correctionCallback(correction);
+
+      if (correctionMessages) {
+        for (const msg of correctionMessages) {
+          SyncMessagesLog.add({
+            from: nodeName,
+            to: storageName,
+            msg,
+          });
+        }
+      }
+
+      return correctionMessages;
     });
   };
 

package/src/tests/testUtils.ts

@@ -40,6 +40,14 @@ export function randomAgentAndSessionID(): [ControlledAgent, SessionID] {
   return [new ControlledAgent(agentSecret, Crypto), sessionID];
 }
 
+export function agentAndSessionIDFromSecret(
+  secret: AgentSecret,
+): [ControlledAgent, SessionID] {
+  const sessionID = Crypto.newRandomSessionID(Crypto.getAgentID(secret));
+
+  return [new ControlledAgent(secret, Crypto), sessionID];
+}
+
 export function nodeWithRandomAgentAndSessionID() {
   const [agent, session] = randomAgentAndSessionID();
   return new LocalNode(agent.agentSecret, session, Crypto);
@@ -154,8 +162,8 @@ export function connectTwoPeers(
   bRole: "client" | "server",
 ) {
   const [aAsPeer, bAsPeer] = connectedPeers(
-    "peer:" + a.getCurrentAgent().id,
-    "peer:" + b.getCurrentAgent().id,
+    "peer:" + a.currentSessionID,
+    "peer:" + b.currentSessionID,
     {
       peer1role: aRole,
       peer2role: bRole,
@@ -443,7 +451,7 @@ export function getSyncServerConnectedPeer(opts: {
 
   const { peer1, peer2 } = connectedPeersWithMessagesTracking({
     peer1: {
-      id: currentSyncServer.getCurrentAgent().id,
+      id: currentSyncServer.currentSessionID,
       role: "server",
       name: opts.syncServerName,
     },
@@ -472,9 +480,13 @@ export function setupTestNode(
   opts: {
     isSyncServer?: boolean;
     connected?: boolean;
+    secret?: AgentSecret;
   } = {},
 ) {
-  const [admin, session] = randomAgentAndSessionID();
+  const [admin, session] = opts.secret
+    ? agentAndSessionIDFromSecret(opts.secret)
+    : randomAgentAndSessionID();
+
   let node = new LocalNode(admin.agentSecret, session, Crypto);
 
   if (opts.isSyncServer) {
@@ -489,7 +501,7 @@ export function setupTestNode(
   }) {
     const { peer, peerStateOnServer, peerOnServer } =
       getSyncServerConnectedPeer({
-        peerId: node.getCurrentAgent().id,
+        peerId: session,
         syncServerName: opts?.syncServerName,
         ourName: opts?.ourName,
         syncServer: opts?.syncServer,
@@ -551,6 +563,13 @@ export function setupTestNode(
 
       return node;
     },
+    spawnNewSession: () => {
+      return setupTestNode({
+        secret: node.agentSecret,
+        connected: opts.connected,
+        isSyncServer: opts.isSyncServer,
+      });
+    },
   };
 
   return ctx;
@@ -638,6 +657,11 @@ export async function setupTestAccount(
     connectToSyncServer,
     addStorage,
     addAsyncStorage,
+    disconnect: () => {
+      ctx.node.syncManager.getPeers().forEach((peer) => {
+        peer.gracefulShutdown();
+      });
+    },
   };
 }
 

package/src/typeUtils/accountOrAgentIDfromSessionID.ts

@@ -1,5 +1,5 @@
-import { RawAccountID } from "../coValues/account.js";
-import { AgentID, SessionID } from "../ids.js";
+import type { RawAccountID } from "../coValues/account.js";
+import type { AgentID, SessionID } from "../ids.js";
 
 export function accountOrAgentIDfromSessionID(
   sessionID: SessionID,

package/src/typeUtils/expectGroup.ts

@@ -1,15 +1,18 @@
-import { type RawCoValue, expectMap } from "../coValue.js";
+import { type RawCoValue } from "../coValue.js";
 import { RawGroup } from "../coValues/group.js";
 
 export function expectGroup(content: RawCoValue): RawGroup {
-  const map = expectMap(content);
-  if (map.core.verified.header.ruleset.type !== "group") {
+  if (content.type !== "comap") {
+    throw new Error("Expected group");
+  }
+
+  if (content.core.verified.header.ruleset.type !== "group") {
     throw new Error("Expected group ruleset in group");
   }
 
-  if (!(map instanceof RawGroup)) {
+  if (!(content instanceof RawGroup)) {
     throw new Error("Expected group");
   }
 
-  return map;
+  return content;
 }