cojson 0.13.5 → 0.13.7

package/src/tests/sync.upload.test.ts

@@ -0,0 +1,202 @@
+import { beforeEach, describe, expect, test } from "vitest";
+
+import { expectMap } from "../coValue";
+import {
+  SyncMessagesLog,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
+
+let jazzCloud = setupTestNode({ isSyncServer: true });
+
+beforeEach(async () => {
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
+describe("client to server upload", () => {
+  test("coValue uploading", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Group sessions: header/3",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("coValue with parent groups uploading", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const parentGroup = client.node.createGroup();
+    parentGroup.addMember("everyone", "reader");
+
+    group.extend(parentGroup);
+
+    const map = group.createMap();
+    map.set("hello", "world");
+
+    await map.core.waitForSync();
+
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("hello")).toEqual("world");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        ParentGroup: parentGroup.core,
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 6",
+        "server -> client | KNOWN ParentGroup sessions: header/6",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | KNOWN Group sessions: header/5",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("wrong optimistic known state should be corrected", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const map = group.createMap({
+      fromServer: "initial",
+      fromClient: "initial",
+    });
+
+    // Load the coValue on the client
+    await map.core.waitForSync();
+
+    // Forcefully delete the coValue from the client (simulating some data loss)
+    jazzCloud.node.coValuesStore.coValues.delete(map.id);
+
+    map.set("fromClient", "updated", "trusting");
+
+    await waitFor(() => {
+      const coValue = expectMap(
+        jazzCloud.node.expectCoValueLoaded(map.id).getCurrentContent(),
+      );
+      expect(coValue.get("fromClient")).toEqual("updated");
+    });
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | KNOWN Group sessions: header/5",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Map sessions: header/1",
+        "client -> server | CONTENT Map header: false new: After: 1 New: 1",
+        "server -> client | KNOWN CORRECTION Map sessions: empty",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | KNOWN Map sessions: header/2",
+      ]
+    `);
+  });
+
+  test("large coValue upload streaming", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    group.addMember("everyone", "writer");
+
+    const largeMap = group.createMap();
+
+    // Generate a large amount of data (about 100MB)
+    const dataSize = 1 * 1024 * 1024;
+    const chunkSize = 1024; // 1KB chunks
+    const chunks = dataSize / chunkSize;
+
+    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
+
+    for (let i = 0; i < chunks; i++) {
+      const key = `key${i}`;
+      largeMap.set(key, value, "trusting");
+    }
+
+    await loadCoValueOrFail(client.node, largeMap.id);
+
+    await largeMap.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: largeMap.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | KNOWN Group sessions: header/5",
+        "client -> server | CONTENT Map header: true new: ",
+        "server -> client | KNOWN Map sessions: header/0",
+        "client -> server | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> client | KNOWN Map sessions: header/73",
+        "client -> server | CONTENT Map header: false new: After: 73 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 146 New: 73",
+        "server -> client | KNOWN Map sessions: header/146",
+        "client -> server | CONTENT Map header: false new: After: 219 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 292 New: 73",
+        "server -> client | KNOWN Map sessions: header/219",
+        "client -> server | CONTENT Map header: false new: After: 365 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 438 New: 73",
+        "server -> client | KNOWN Map sessions: header/292",
+        "client -> server | CONTENT Map header: false new: After: 511 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 584 New: 73",
+        "server -> client | KNOWN Map sessions: header/365",
+        "client -> server | CONTENT Map header: false new: After: 657 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 730 New: 73",
+        "server -> client | KNOWN Map sessions: header/438",
+        "client -> server | CONTENT Map header: false new: After: 803 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 876 New: 73",
+        "server -> client | KNOWN Map sessions: header/511",
+        "client -> server | CONTENT Map header: false new: After: 949 New: 73",
+        "client -> server | CONTENT Map header: false new: After: 1022 New: 2",
+        "server -> client | KNOWN Map sessions: header/584",
+        "server -> client | KNOWN Map sessions: header/657",
+        "server -> client | KNOWN Map sessions: header/730",
+        "server -> client | KNOWN Map sessions: header/803",
+        "server -> client | KNOWN Map sessions: header/876",
+        "server -> client | KNOWN Map sessions: header/949",
+        "server -> client | KNOWN Map sessions: header/1022",
+        "server -> client | KNOWN Map sessions: header/1024",
+      ]
+    `);
+  });
+});

package/src/tests/testUtils.ts

@@ -8,12 +8,13 @@ import {
 import { expect, onTestFinished, vi } from "vitest";
 import { ControlledAgent } from "../coValues/account.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
-import type { CoID, RawCoValue } from "../exports.js";
+import type { CoID, CoValueCore, RawCoValue } from "../exports.js";
 import type { SessionID } from "../ids.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import type { Peer, SyncMessage } from "../sync.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
+import { toSimplifiedMessages } from "./messagesTestUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -234,11 +235,13 @@ export function shouldNotResolve<T>(
   });
 }
 
-export function waitFor(callback: () => boolean | void) {
+export function waitFor(
+  callback: () => boolean | void | Promise<boolean | void>,
+) {
   return new Promise<void>((resolve, reject) => {
-    const checkPassed = () => {
+    const checkPassed = async () => {
       try {
-        return { ok: callback(), error: null };
+        return { ok: await callback(), error: null };
       } catch (error) {
         return { ok: false, error };
       }
@@ -246,8 +249,8 @@ export function waitFor(callback: () => boolean | void) {
 
     let retries = 0;
 
-    const interval = setInterval(() => {
-      const { ok, error } = checkPassed();
+    const interval = setInterval(async () => {
+      const { ok, error } = await checkPassed();
 
       if (ok !== false) {
         clearInterval(interval);
@@ -378,89 +381,238 @@ export function tearDownTestMetricReader() {
   metrics.disable();
 }
 
-export function setupSyncServer() {
-  syncServer.current = createTestNode();
-  return syncServer.current;
-}
+export class SyncMessagesLog {
+  static messages: SyncTestMessage[] = [];
 
-export async function createConnectedTestAgentNode(opts = { connected: true }) {
-  if (!syncServer.current) {
-    throw new Error("Sync server not initialized");
+  static add(message: SyncTestMessage) {
+    this.messages.push(message);
   }
 
-  const [admin, session] = randomAnonymousAccountAndSessionID();
-  const node = new LocalNode(admin, session, Crypto);
+  static clear() {
+    this.messages.length = 0;
+  }
 
-  const { nodeToServerPeer, serverToNodePeer, messages, addServerPeer } =
-    connectNodeToSyncServer(node, opts.connected);
+  static getMessages(coValueMapping: { [key: string]: CoValueCore }) {
+    return toSimplifiedMessages(coValueMapping, SyncMessagesLog.messages);
+  }
 
-  return { node, nodeToServerPeer, serverToNodePeer, messages, addServerPeer };
+  static debugMessages(coValueMapping: { [key: string]: CoValueCore }) {
+    console.log(SyncMessagesLog.getMessages(coValueMapping));
+  }
 }
 
-export async function createConnectedTestNode(opts = { connected: true }) {
-  if (!syncServer.current) {
-    throw new Error("Sync server not initialized");
+export function setupTestNode(
+  opts: {
+    isSyncServer?: boolean;
+    connected?: boolean;
+  } = {},
+) {
+  const [admin, session] = randomAnonymousAccountAndSessionID();
+  let node = new LocalNode(admin, session, Crypto);
+
+  if (opts.isSyncServer) {
+    syncServer.current = node;
   }
 
+  function connectToSyncServer(opts?: {
+    syncServerName?: string;
+    ourName?: string;
+    syncServer?: LocalNode;
+  }) {
+    const currentSyncServer = opts?.syncServer ?? syncServer.current;
+
+    if (!currentSyncServer) {
+      throw new Error("Sync server not initialized");
+    }
+
+    if (currentSyncServer.account.id === node.account.id) {
+      throw new Error("Cannot connect to self");
+    }
+
+    const { peer1, peer2 } = connectedPeersWithMessagesTracking({
+      peer1: {
+        id: currentSyncServer.account.id,
+        role: "server",
+        name: opts?.syncServerName,
+      },
+      peer2: { id: node.account.id, role: "client", name: opts?.ourName },
+    });
+
+    node.syncManager.addPeer(peer1);
+    currentSyncServer.syncManager.addPeer(peer2);
+
+    return {
+      peerState: node.syncManager.peers[peer1.id]!,
+      peer: peer1,
+      peerStateOnServer: currentSyncServer.syncManager.peers[peer2.id]!,
+      peerOnServer: peer2,
+    };
+  }
+
+  function addStoragePeer(opts: { ourName?: string } = {}) {
+    const storage = createTestNode();
+
+    const { peer1, peer2 } = connectedPeersWithMessagesTracking({
+      peer1: { id: storage.account.id, role: "storage" },
+      peer2: { id: node.account.id, role: "client", name: opts.ourName },
+    });
+
+    peer1.priority = 100;
+
+    node.syncManager.addPeer(peer1);
+    storage.syncManager.addPeer(peer2);
+
+    return {
+      storage,
+      peer: node.syncManager.peers[peer1.id]!,
+    };
+  }
+
+  if (opts.connected) {
+    connectToSyncServer();
+  }
+
+  return {
+    node,
+    connectToSyncServer,
+    addStoragePeer,
+    restart: () => {
+      node.gracefulShutdown();
+      node = new LocalNode(admin, session, Crypto);
+
+      if (opts.isSyncServer) {
+        syncServer.current = node;
+      }
+
+      return node;
+    },
+  };
+}
+
+export async function setupTestAccount(
+  opts: {
+    isSyncServer?: boolean;
+    connected?: boolean;
+  } = {},
+) {
   const ctx = await LocalNode.withNewlyCreatedAccount({
     peersToLoadFrom: [],
     crypto: Crypto,
     creationProps: { name: "Client" },
   });
 
-  const { nodeToServerPeer, serverToNodePeer, messages, addServerPeer } =
-    connectNodeToSyncServer(ctx.node, opts.connected);
+  if (opts.isSyncServer) {
+    syncServer.current = ctx.node;
+  }
+
+  function connectToSyncServer(opts?: {
+    syncServerName?: string;
+    ourName?: string;
+    syncServer?: LocalNode;
+  }) {
+    const currentSyncServer = opts?.syncServer ?? syncServer.current;
+
+    if (!currentSyncServer) {
+      throw new Error("Sync server not initialized");
+    }
+
+    if (currentSyncServer.account.id === ctx.node.account.id) {
+      throw new Error("Cannot connect to self");
+    }
+
+    const { peer1, peer2 } = connectedPeersWithMessagesTracking({
+      peer1: {
+        id: currentSyncServer.account.id,
+        role: "server",
+        name: opts?.syncServerName,
+      },
+      peer2: { id: ctx.node.account.id, role: "client", name: opts?.ourName },
+    });
+
+    ctx.node.syncManager.addPeer(peer1);
+    currentSyncServer.syncManager.addPeer(peer2);
+
+    function getCurrentPeerState() {
+      return ctx.node.syncManager.peers[peer1.id]!;
+    }
+
+    return {
+      peerState: getCurrentPeerState(),
+      peer: peer1,
+      peerStateOnServer: currentSyncServer.syncManager.peers[peer2.id]!,
+      peerOnServer: peer2,
+      getCurrentPeerState,
+    };
+  }
+
+  function addStoragePeer(opts: { ourName?: string } = {}) {
+    const storage = createTestNode();
+
+    const { peer1, peer2 } = connectedPeersWithMessagesTracking({
+      peer1: { id: storage.account.id, role: "storage" },
+      peer2: { id: ctx.node.account.id, role: "client", name: opts.ourName },
+    });
+
+    peer1.priority = 100;
+
+    ctx.node.syncManager.addPeer(peer1);
+    storage.syncManager.addPeer(peer2);
+
+    return {
+      storage,
+      peer: ctx.node.syncManager.peers[peer1.id]!,
+    };
+  }
+
+  if (opts.connected) {
+    connectToSyncServer();
+  }
 
   return {
     node: ctx.node,
     accountID: ctx.accountID,
-    nodeToServerPeer,
-    serverToNodePeer,
-    messages,
-    addServerPeer,
+    connectToSyncServer,
+    addStoragePeer,
   };
 }
 
-export function connectNodeToSyncServer(node: LocalNode, connected = true) {
-  if (!syncServer.current) {
-    throw new Error("Sync server not initialized");
-  }
-
-  const [nodeToServerPeer, serverToNodePeer] = connectedPeers(
-    syncServer.current.account.id,
-    node.account.id,
-    {
-      peer1role: "server",
-      peer2role: "client",
-    },
-  );
+export type SyncTestMessage = {
+  from: string;
+  to: string;
+  msg: SyncMessage;
+};
 
-  const messages: {
-    from: "client" | "server";
-    msg: SyncMessage;
-  }[] = [];
+export function connectedPeersWithMessagesTracking(opts: {
+  peer1: { id: string; role: Peer["role"]; name?: string };
+  peer2: { id: string; role: Peer["role"]; name?: string };
+}) {
+  const [peer1, peer2] = connectedPeers(opts.peer1.id, opts.peer2.id, {
+    peer1role: opts.peer1.role,
+    peer2role: opts.peer2.role,
+  });
 
-  const serverPush = serverToNodePeer.outgoing.push;
-  serverToNodePeer.outgoing.push = (msg) => {
-    messages.push({ from: "server", msg });
-    return serverPush.call(serverToNodePeer.outgoing, msg);
+  const peer1Push = peer1.outgoing.push;
+  peer1.outgoing.push = (msg) => {
+    SyncMessagesLog.add({
+      from: opts.peer2.name ?? opts.peer2.role,
+      to: opts.peer1.name ?? opts.peer1.role,
+      msg,
+    });
+    return peer1Push.call(peer1.outgoing, msg);
   };
 
-  const clientPush = nodeToServerPeer.outgoing.push;
-  nodeToServerPeer.outgoing.push = (msg) => {
-    messages.push({ from: "client", msg });
-    return clientPush.call(nodeToServerPeer.outgoing, msg);
+  const peer2Push = peer2.outgoing.push;
+  peer2.outgoing.push = (msg) => {
+    SyncMessagesLog.add({
+      from: opts.peer1.name ?? opts.peer1.role,
+      to: opts.peer2.name ?? opts.peer2.role,
+      msg,
+    });
+    return peer2Push.call(peer2.outgoing, msg);
   };
 
-  syncServer.current.syncManager.addPeer(serverToNodePeer);
-  if (connected) {
-    node.syncManager.addPeer(nodeToServerPeer);
-  }
-
   return {
-    nodeToServerPeer,
-    serverToNodePeer,
-    messages,
-    addServerPeer: () => node.syncManager.addPeer(nodeToServerPeer),
+    peer1,
+    peer2,
   };
 }