cojson 0.15.8 → 0.15.10

package/src/tests/sync.storageAsync.test.ts

@@ -0,0 +1,486 @@
+import { assert, beforeEach, describe, expect, test, vi } from "vitest";
+
+import { emptyKnownState } from "../exports";
+import {
+  SyncMessagesLog,
+  TEST_NODE_CONFIG,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
+
+// We want to simulate a real world communication that happens asynchronously
+TEST_NODE_CONFIG.withAsyncPeers = true;
+
+describe("client with storage syncs with server", () => {
+  let jazzCloud: ReturnType<typeof setupTestNode>;
+
+  beforeEach(async () => {
+    SyncMessagesLog.clear();
+    jazzCloud = setupTestNode({
+      isSyncServer: true,
+    });
+  });
+
+  test("coValue loading (empty storage)", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+    await client.addAsyncStorage();
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | LOAD Map sessions: empty",
+        "storage -> client | KNOWN Map sessions: empty",
+        "client -> server | LOAD Map sessions: empty",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | KNOWN Group sessions: header/3",
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | KNOWN Map sessions: header/1",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 1",
+      ]
+    `);
+  });
+
+  test("coValue loading (synced storage)", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+    const { storage } = await client.addAsyncStorage();
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    const firstLoad = await loadCoValueOrFail(client.node, map.id);
+    await firstLoad.core.waitForSync(); // Need to wait for sync with storage
+
+    client.restart();
+
+    client.connectToSyncServer();
+    client.addStorage({
+      ourName: "client",
+      storage,
+    });
+
+    SyncMessagesLog.clear();
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | LOAD Map sessions: empty",
+        "storage -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | LOAD Group sessions: header/3",
+        "storage -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | LOAD Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("coValue with parent groups loading", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+    await client.addAsyncStorage();
+
+    const group = jazzCloud.node.createGroup();
+    const parentGroup = jazzCloud.node.createGroup();
+    parentGroup.addMember("everyone", "reader");
+
+    group.extend(parentGroup);
+
+    const map = group.createMap();
+    map.set("hello", "world");
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        ParentGroup: parentGroup.core,
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | LOAD Map sessions: empty",
+        "storage -> client | KNOWN Map sessions: empty",
+        "client -> server | LOAD Map sessions: empty",
+        "server -> client | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | KNOWN ParentGroup sessions: header/6",
+        "client -> storage | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "client -> server | KNOWN Group sessions: header/5",
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | KNOWN Map sessions: header/1",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 1",
+      ]
+    `);
+  });
+
+  test("updating a coValue while offline", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+    await client.addAsyncStorage();
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+
+    client.node.syncManager.getPeers()[0]?.gracefulShutdown();
+
+    SyncMessagesLog.clear();
+    map.set("hello", "updated", "trusting");
+
+    client.connectToSyncServer();
+
+    await map.core.waitForSync();
+
+    expect(mapOnClient.get("hello")).toEqual("updated");
+
+    await mapOnClient.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | LOAD Group sessions: header/3",
+        "client -> server | LOAD Map sessions: header/1",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | CONTENT Map header: false new: After: 1 New: 1",
+        "client -> server | KNOWN Group sessions: header/3",
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | KNOWN Map sessions: header/2",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 2",
+        "client -> server | KNOWN Map sessions: header/2",
+        "client -> storage | CONTENT Map header: false new: After: 1 New: 1",
+      ]
+    `);
+  });
+});
+
+describe("client syncs with a server with storage", () => {
+  let jazzCloud: ReturnType<typeof setupTestNode>;
+
+  beforeEach(async () => {
+    SyncMessagesLog.clear();
+    jazzCloud = setupTestNode({
+      isSyncServer: true,
+    });
+    jazzCloud.addStorage({
+      ourName: "server",
+    });
+  });
+
+  test("coValue uploading", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Map sessions: header/1",
+        "server -> storage | CONTENT Map header: true new: After: 0 New: 1",
+      ]
+    `);
+  });
+
+  test("large coValue streaming", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer({
+      syncServer: jazzCloud.node,
+    });
+
+    const { storage } = await client.addAsyncStorage({
+      ourName: "client",
+    });
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const largeMap = group.createMap();
+
+    // Generate a large amount of data (about 100MB)
+    const dataSize = 1 * 200 * 1024;
+    const chunkSize = 1024; // 1KB chunks
+    const chunks = dataSize / chunkSize;
+
+    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
+
+    for (let i = 0; i < chunks; i++) {
+      const key = `key${i}`;
+      largeMap.set(key, value, "trusting");
+    }
+
+    await largeMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: largeMap.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
+        "client -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> storage | CONTENT Map header: false new: After: 146 New: 54",
+        "client -> server | CONTENT Map header: true new:  expectContentUntil: header/200",
+        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 146 New: 54",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> storage | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | KNOWN Map sessions: header/0",
+        "server -> storage | CONTENT Map header: true new:  expectContentUntil: header/200",
+        "server -> client | KNOWN Map sessions: header/73",
+        "server -> storage | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> client | KNOWN Map sessions: header/146",
+        "server -> storage | CONTENT Map header: false new: After: 73 New: 73",
+        "server -> client | KNOWN Map sessions: header/200",
+        "server -> storage | CONTENT Map header: false new: After: 146 New: 54",
+      ]
+    `);
+
+    SyncMessagesLog.clear();
+
+    client.restart();
+
+    client.connectToSyncServer({
+      ourName: "client",
+      syncServer: jazzCloud.node,
+    });
+
+    client.addStorage({
+      ourName: "client",
+      storage,
+    });
+
+    const mapOnClient2 = await loadCoValueOrFail(client.node, largeMap.id);
+
+    await mapOnClient2.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: largeMap.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | LOAD Map sessions: empty",
+        "storage -> client | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | LOAD Group sessions: header/5",
+        "storage -> client | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/200",
+        "client -> server | LOAD Map sessions: header/200",
+        "storage -> client | CONTENT Map header: true new: After: 73 New: 73",
+        "storage -> client | CONTENT Map header: true new: After: 146 New: 54",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> client | KNOWN Map sessions: header/200",
+      ]
+    `);
+  });
+
+  test("storing stale data should not compromise the signatures", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer({
+      syncServer: jazzCloud.node,
+    });
+
+    const { storage } = await client.addAsyncStorage({
+      ourName: "client",
+    });
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const largeMap = group.createMap();
+
+    // Generate a large amount of data (about 100MB)
+    const dataSize = 1 * 200 * 1024;
+    const chunkSize = 1024; // 1KB chunks
+    const chunks = dataSize / chunkSize;
+
+    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
+
+    for (let i = 0; i < chunks; i++) {
+      const key = `key${i}`;
+      largeMap.set(key, value, "trusting");
+    }
+
+    await largeMap.core.waitForSync();
+
+    const newContentChunks = largeMap.core.verified.newContentSince(
+      emptyKnownState(largeMap.id),
+    );
+
+    assert(newContentChunks);
+    assert(newContentChunks.length > 1);
+
+    const correctionSpy = vi.fn();
+
+    client.node.storage?.store(newContentChunks.slice(1, 2), correctionSpy);
+
+    // Wait for the content to be stored in the storage
+    // We can't use waitForSync because we are trying to store stale data
+    await new Promise((resolve) => setTimeout(resolve, 100));
+
+    expect(correctionSpy).not.toHaveBeenCalled();
+
+    client.restart();
+
+    client.connectToSyncServer({
+      ourName: "client",
+      syncServer: jazzCloud.node,
+    });
+
+    client.addStorage({
+      ourName: "client",
+      storage,
+    });
+
+    const mapOnClient2 = await loadCoValueOrFail(client.node, largeMap.id);
+
+    await waitFor(async () => {
+      expect(mapOnClient2.core.knownState()).toEqual(
+        largeMap.core.knownState(),
+      );
+    });
+  });
+
+  test("large coValue streaming from cold server", async () => {
+    const server = setupTestNode({
+      isSyncServer: true,
+    });
+    const { storage: serverStorage } = server.addStorage({
+      ourName: "server",
+    });
+
+    const client = setupTestNode();
+
+    client.connectToSyncServer({
+      syncServer: server.node,
+    });
+
+    const { storage } = await client.addAsyncStorage({
+      ourName: "client",
+    });
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const largeMap = group.createMap();
+
+    // Generate a large amount of data (about 100MB)
+    const dataSize = 1 * 200 * 1024;
+    const chunkSize = 1024; // 1KB chunks
+    const chunks = dataSize / chunkSize;
+
+    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
+
+    for (let i = 0; i < chunks; i++) {
+      const key = `key${i}`;
+      largeMap.set(key, value, "trusting");
+    }
+
+    await largeMap.core.waitForSync();
+
+    SyncMessagesLog.clear();
+
+    server.restart();
+
+    server.addStorage({
+      ourName: "server",
+      storage: serverStorage,
+    });
+
+    client.restart();
+
+    client.connectToSyncServer({
+      ourName: "client",
+      syncServer: server.node,
+    });
+
+    client.addStorage({
+      ourName: "client",
+      storage,
+    });
+
+    const mapOnClient2 = await loadCoValueOrFail(client.node, largeMap.id);
+
+    await mapOnClient2.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: largeMap.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> storage | LOAD Map sessions: empty",
+        "storage -> client | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | LOAD Group sessions: header/5",
+        "storage -> client | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/200",
+        "client -> server | LOAD Map sessions: header/200",
+        "storage -> client | CONTENT Map header: true new: After: 73 New: 73",
+        "storage -> client | CONTENT Map header: true new: After: 146 New: 54",
+        "server -> storage | LOAD Group sessions: empty",
+        "storage -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> storage | LOAD Map sessions: empty",
+        "storage -> server | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/200",
+        "server -> client | KNOWN Map sessions: header/200",
+      ]
+    `);
+  });
+});

package/src/tests/sync.test.ts

@@ -9,30 +9,29 @@ import {
 } from "vitest";
 import { expectMap } from "../coValue.js";
 import { RawCoMap } from "../coValues/coMap.js";
-import type { RawGroup } from "../coValues/group.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
-import { LocalNode } from "../localNode.js";
 import { connectedPeers, newQueuePair } from "../streamUtils.js";
 import type { LoadMessage } from "../sync.js";
 import {
+  TEST_NODE_CONFIG,
   blockMessageTypeOnOutgoingPeer,
   connectTwoPeers,
   createTestMetricReader,
   createTestNode,
   loadCoValueOrFail,
   nodeWithRandomAgentAndSessionID,
-  randomAgentAndSessionID,
   setupTestAccount,
   setupTestNode,
   tearDownTestMetricReader,
   waitFor,
 } from "./testUtils.js";
 
+// We want to simulate a real world communication that happens asynchronously
+TEST_NODE_CONFIG.withAsyncPeers = true;
+
 const Crypto = await WasmCrypto.create();
 
-let jazzCloud = setupTestNode({
-  isSyncServer: true,
-});
+let jazzCloud: ReturnType<typeof setupTestNode>;
 
 beforeEach(async () => {
   jazzCloud = setupTestNode({
@@ -47,16 +46,14 @@ test("If we add a client peer, but it never subscribes to a coValue, it won't ge
 
   const map = group.createMap();
 
-  const [inRx, _inTx] = newQueuePair();
-  const [outRx, outTx] = newQueuePair();
-  const outRxQ = outRx[Symbol.asyncIterator]();
+  const [inRx] = newQueuePair();
+  const [, outTx] = newQueuePair();
 
   node.syncManager.addPeer({
     id: "test",
     incoming: inRx,
     outgoing: outTx,
     role: "client",
-    crashOnClose: true,
   });
 
   map.set("hello", "world", "trusting");
@@ -66,7 +63,9 @@ test("If we add a client peer, but it never subscribes to a coValue, it won't ge
   );
 
   const result = await Promise.race([
-    outRxQ.next().then((value) => value.value),
+    new Promise((resolve) => {
+      inRx.onMessage((msg) => resolve(msg));
+    }),
     timeoutPromise,
   ]);
 
@@ -115,7 +114,7 @@ test("Can sync a coValue with private transactions through a server to another c
 test("should keep the peer state when the peer closes", async () => {
   const client = setupTestNode();
 
-  const { peer, peerState } = client.connectToSyncServer();
+  const { peer, peerState, peerOnServer } = client.connectToSyncServer();
 
   const group = jazzCloud.node.createGroup();
   const map = group.createMap();
@@ -125,10 +124,11 @@ test("should keep the peer state when the peer closes", async () => {
 
   const syncManager = client.node.syncManager;
 
-  // @ts-expect-error Simulating a peer closing, leveraging the direct connection between the client/server peers
-  await peer.outgoing.push("Disconnected");
+  peerOnServer.outgoing.push("Disconnected");
 
-  await waitFor(() => peerState?.closed);
+  await waitFor(() => {
+    return peerState.closed;
+  });
 
   expect(syncManager.peers[peer.id]).not.toBeUndefined();
 });
@@ -136,7 +136,7 @@ test("should keep the peer state when the peer closes", async () => {
 test("should delete the peer state when the peer closes if deletePeerStateOnClose is true", async () => {
   const client = setupTestNode();
 
-  const { peer, peerState } = client.connectToSyncServer();
+  const { peer, peerState, peerOnServer } = client.connectToSyncServer();
 
   peer.deletePeerStateOnClose = true;
 
@@ -148,8 +148,7 @@ test("should delete the peer state when the peer closes if deletePeerStateOnClos
 
   const syncManager = client.node.syncManager;
 
-  // @ts-expect-error Simulating a peer closing, leveraging the direct connection between the client/server peers
-  await peer.outgoing.push("Disconnected");
+  peerOnServer.outgoing.push("Disconnected");
 
   await waitFor(() => peerState?.closed);
 
@@ -929,7 +928,6 @@ describe("metrics", () => {
       incoming: inPeer1,
       outgoing: outPeer1,
       role: "client",
-      crashOnClose: false,
     });
 
     connectedPeers = await metricReader.getMetricValue("jazz.peers", {
@@ -944,7 +942,6 @@ describe("metrics", () => {
       incoming: inPeer2,
       outgoing: outPeer2,
       role: "client",
-      crashOnClose: false,
     });
 
     connectedPeers = await metricReader.getMetricValue("jazz.peers", {
@@ -959,7 +956,6 @@ describe("metrics", () => {
       incoming: inServer1,
       outgoing: outServer1,
       role: "server",
-      crashOnClose: false,
     });
     connectedServerPeers = await metricReader.getMetricValue("jazz.peers", {
       role: "server",
@@ -970,7 +966,6 @@ describe("metrics", () => {
     });
     expect(connectedPeers).toBe(2);
 
-    // @ts-expect-error Simulating peer-1 closing
     await outPeer1.push("Disconnected");
     await waitFor(() => node.syncManager.peers["peer-1"]?.closed);
 
@@ -979,7 +974,6 @@ describe("metrics", () => {
     });
     expect(connectedPeers).toBe(1);
 
-    // @ts-expect-error Simulating server-1 closing
     await outServer1.push("Disconnected");
 
     await waitFor(() => node.syncManager.peers["server-1"]?.closed);