cojson 0.15.8 → 0.15.10

package/src/tests/sync.upload.test.ts

@@ -4,13 +4,17 @@ import { expectList, expectMap } from "../coValue";
 import { WasmCrypto } from "../crypto/WasmCrypto";
 import {
   SyncMessagesLog,
+  TEST_NODE_CONFIG,
   loadCoValueOrFail,
   setupTestNode,
   waitFor,
 } from "./testUtils";
 
+// We want to simulate a real world communication that happens asynchronously
+TEST_NODE_CONFIG.withAsyncPeers = true;
+
 const Crypto = await WasmCrypto.create();
-let jazzCloud = setupTestNode({ isSyncServer: true });
+let jazzCloud: ReturnType<typeof setupTestNode>;
 
 beforeEach(async () => {
   SyncMessagesLog.clear();
@@ -40,8 +44,8 @@ describe("client to server upload", () => {
     ).toMatchInlineSnapshot(`
       [
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | KNOWN Group sessions: header/3",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN Map sessions: header/1",
       ]
     `);
@@ -75,10 +79,10 @@ describe("client to server upload", () => {
     ).toMatchInlineSnapshot(`
       [
         "client -> server | CONTENT ParentGroup header: true new: After: 0 New: 6",
-        "server -> client | KNOWN ParentGroup sessions: header/6",
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Group sessions: header/5",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN ParentGroup sessions: header/6",
+        "server -> client | KNOWN Group sessions: header/5",
         "server -> client | KNOWN Map sessions: header/1",
       ]
     `);
@@ -113,8 +117,8 @@ describe("client to server upload", () => {
     ).toMatchInlineSnapshot(`
       [
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
-        "server -> client | KNOWN Group sessions: header/3",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN Map sessions: header/1",
         "client -> server | CONTENT Map header: false new: After: 1 New: 1",
         "server -> client | KNOWN Map sessions: header/2",
@@ -158,8 +162,8 @@ describe("client to server upload", () => {
     ).toMatchInlineSnapshot(`
       [
         "client -> server | CONTENT Group header: true new: After: 0 New: 5",
-        "server -> client | KNOWN Group sessions: header/5",
         "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/5",
         "server -> client | KNOWN Map sessions: header/1",
         "client -> server | CONTENT Map header: false new: After: 1 New: 1",
         "server -> client | KNOWN CORRECTION Map sessions: empty",
@@ -233,16 +237,17 @@ describe("client to server upload", () => {
         "otherClient -> server | KNOWN Colist sessions: header/1",
         "client -> server | CONTENT Colist header: false new: After: 1 New: 1",
         "server -> client | KNOWN Colist sessions: header/2",
+        "server -> otherClient | CONTENT Colist header: false new: After: 1 New: 1",
         "otherClient -> server | LOAD Colist sessions: header/3",
         "client -> server | CONTENT Colist header: false new: After: 2 New: 1",
         "server -> otherClient | CONTENT Colist header: false new: After: 1 New: 1",
         "server -> client | KNOWN Colist sessions: header/3",
-        "otherClient -> server | KNOWN Colist sessions: header/4",
         "server -> otherClient | CONTENT Colist header: false new: After: 2 New: 1",
+        "otherClient -> server | KNOWN Colist sessions: header/4",
         "otherClient -> server | CONTENT Colist header: false new: After: 0 New: 2",
+        "otherClient -> server | KNOWN Colist sessions: header/5",
         "server -> otherClient | KNOWN Colist sessions: header/5",
         "server -> client | CONTENT Colist header: false new: After: 0 New: 2",
-        "otherClient -> server | KNOWN Colist sessions: header/5",
         "client -> server | KNOWN Colist sessions: header/5",
       ]
     `);
@@ -283,38 +288,38 @@ describe("client to server upload", () => {
       [
         "client -> server | LOAD Map sessions: empty",
         "server -> client | CONTENT Group header: true new: After: 0 New: 5",
+        "server -> client | CONTENT Map header: true new:  expectContentUntil: header/1024",
+        "server -> client | CONTENT Map header: false new: After: 0 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 73 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 146 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 219 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 292 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 365 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 438 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 511 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 584 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 657 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 730 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 803 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 876 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 949 New: 73",
+        "server -> client | CONTENT Map header: false new: After: 1022 New: 2",
         "client -> server | KNOWN Group sessions: header/5",
-        "server -> client | CONTENT Map header: true new: ",
         "client -> server | KNOWN Map sessions: header/0",
-        "server -> client | CONTENT Map header: false new: After: 0 New: 73",
         "client -> server | KNOWN Map sessions: header/73",
-        "server -> client | CONTENT Map header: false new: After: 73 New: 73",
         "client -> server | KNOWN Map sessions: header/146",
-        "server -> client | CONTENT Map header: false new: After: 146 New: 73",
         "client -> server | KNOWN Map sessions: header/219",
-        "server -> client | CONTENT Map header: false new: After: 219 New: 73",
         "client -> server | KNOWN Map sessions: header/292",
-        "server -> client | CONTENT Map header: false new: After: 292 New: 73",
         "client -> server | KNOWN Map sessions: header/365",
-        "server -> client | CONTENT Map header: false new: After: 365 New: 73",
         "client -> server | KNOWN Map sessions: header/438",
-        "server -> client | CONTENT Map header: false new: After: 438 New: 73",
         "client -> server | KNOWN Map sessions: header/511",
-        "server -> client | CONTENT Map header: false new: After: 511 New: 73",
         "client -> server | KNOWN Map sessions: header/584",
-        "server -> client | CONTENT Map header: false new: After: 584 New: 73",
         "client -> server | KNOWN Map sessions: header/657",
-        "server -> client | CONTENT Map header: false new: After: 657 New: 73",
         "client -> server | KNOWN Map sessions: header/730",
-        "server -> client | CONTENT Map header: false new: After: 730 New: 73",
         "client -> server | KNOWN Map sessions: header/803",
-        "server -> client | CONTENT Map header: false new: After: 803 New: 73",
         "client -> server | KNOWN Map sessions: header/876",
-        "server -> client | CONTENT Map header: false new: After: 876 New: 73",
         "client -> server | KNOWN Map sessions: header/949",
-        "server -> client | CONTENT Map header: false new: After: 949 New: 73",
         "client -> server | KNOWN Map sessions: header/1022",
-        "server -> client | CONTENT Map header: false new: After: 1022 New: 2",
         "client -> server | KNOWN Map sessions: header/1024",
       ]
     `);

package/src/tests/testStorage.ts

@@ -0,0 +1,216 @@
+import { randomUUID } from "node:crypto";
+import { unlinkSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import Database, { type Database as DatabaseT } from "libsql";
+import { onTestFinished } from "vitest";
+import { RawCoID, StorageAPI } from "../exports";
+import {
+  SQLiteDatabaseDriver,
+  StorageApiAsync,
+  StorageApiSync,
+} from "../storage";
+import { getSqliteStorage } from "../storage/sqlite";
+import {
+  SQLiteDatabaseDriverAsync,
+  getSqliteStorageAsync,
+} from "../storage/sqliteAsync";
+import { SyncMessagesLog, SyncTestMessage } from "./testUtils";
+
+class LibSQLSqliteAsyncDriver implements SQLiteDatabaseDriverAsync {
+  private readonly db: DatabaseT;
+
+  constructor(filename: string) {
+    this.db = new Database(filename, {});
+  }
+
+  async initialize() {
+    await this.db.pragma("journal_mode = WAL");
+  }
+
+  async run(sql: string, params: unknown[]) {
+    this.db.prepare(sql).run(params);
+  }
+
+  async query<T>(sql: string, params: unknown[]): Promise<T[]> {
+    return this.db.prepare(sql).all(params) as T[];
+  }
+
+  async get<T>(sql: string, params: unknown[]): Promise<T | undefined> {
+    return this.db.prepare(sql).get(params) as T | undefined;
+  }
+
+  async transaction(callback: () => unknown) {
+    await this.run("BEGIN TRANSACTION", []);
+
+    try {
+      await callback();
+      await this.run("COMMIT", []);
+    } catch (error) {
+      await this.run("ROLLBACK", []);
+    }
+  }
+
+  async closeDb() {
+    this.db.close();
+  }
+}
+
+class LibSQLSqliteSyncDriver implements SQLiteDatabaseDriver {
+  private readonly db: DatabaseT;
+
+  constructor(filename: string) {
+    this.db = new Database(filename, {});
+  }
+
+  initialize() {
+    this.db.pragma("journal_mode = WAL");
+  }
+
+  run(sql: string, params: unknown[]) {
+    this.db.prepare(sql).run(params);
+  }
+
+  query<T>(sql: string, params: unknown[]): T[] {
+    return this.db.prepare(sql).all(params) as T[];
+  }
+
+  get<T>(sql: string, params: unknown[]): T | undefined {
+    return this.db.prepare(sql).get(params) as T | undefined;
+  }
+
+  transaction(callback: () => unknown) {
+    this.run("BEGIN TRANSACTION", []);
+
+    try {
+      callback();
+      this.run("COMMIT", []);
+    } catch (error) {
+      this.run("ROLLBACK", []);
+    }
+  }
+
+  closeDb() {
+    this.db.close();
+  }
+}
+
+export async function createAsyncStorage({
+  filename,
+  nodeName = "client",
+  storageName = "storage",
+}: { filename?: string; nodeName: string; storageName: string }) {
+  const storage = await getSqliteStorageAsync(
+    new LibSQLSqliteAsyncDriver(getDbPath(filename)),
+  );
+
+  onTestFinished(() => {
+    storage.close();
+  });
+
+  trackStorageMessages(storage, nodeName, storageName);
+
+  return storage;
+}
+
+export function createSyncStorage({
+  filename,
+  nodeName = "client",
+  storageName = "storage",
+}: { filename?: string; nodeName: string; storageName: string }) {
+  const storage = getSqliteStorage(
+    new LibSQLSqliteSyncDriver(getDbPath(filename)),
+  );
+
+  trackStorageMessages(storage, nodeName, storageName);
+
+  return storage;
+}
+
+function getDbPath(defaultDbPath?: string) {
+  const dbPath = defaultDbPath ?? join(tmpdir(), `test-${randomUUID()}.db`);
+
+  if (!defaultDbPath) {
+    onTestFinished(() => {
+      unlinkSync(dbPath);
+    });
+  }
+
+  return dbPath;
+}
+
+function trackStorageMessages(
+  storage: StorageAPI,
+  nodeName: string,
+  storageName: string,
+) {
+  const originalStore = storage.store;
+  const originalLoad = storage.load;
+
+  storage.store = function (data, correctionCallback) {
+    for (const msg of data ?? []) {
+      SyncMessagesLog.add({
+        from: nodeName,
+        to: storageName,
+        msg,
+      });
+    }
+
+    return originalStore.call(storage, data, (correction) => {
+      SyncMessagesLog.add({
+        from: storageName,
+        to: nodeName,
+        msg: {
+          action: "known",
+          isCorrection: true,
+          ...correction,
+        },
+      });
+
+      return correctionCallback(correction);
+    });
+  };
+
+  storage.load = function (id, callback, done) {
+    SyncMessagesLog.add({
+      from: nodeName,
+      to: storageName,
+      msg: {
+        action: "load",
+        id: id as RawCoID,
+        sessions: {},
+        header: false,
+      },
+    });
+
+    return originalLoad.call(
+      storage,
+      id,
+      (msg) => {
+        SyncMessagesLog.add({
+          from: storageName,
+          to: nodeName,
+          msg,
+        });
+
+        return callback(msg);
+      },
+      (found) => {
+        if (!found) {
+          SyncMessagesLog.add({
+            from: storageName,
+            to: nodeName,
+            msg: {
+              action: "known",
+              id: id as RawCoID,
+              sessions: {},
+              header: false,
+            },
+          });
+        }
+
+        return done?.(found);
+      },
+    );
+  };
+}

package/src/tests/testUtils.ts

@@ -14,13 +14,15 @@ import {
   type CoValueCore,
   type RawAccount,
   type RawCoValue,
+  StorageAPI,
 } from "../exports.js";
-import type { SessionID } from "../ids.js";
+import type { RawCoID, SessionID } from "../ids.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import type { Peer, SyncMessage } from "../sync.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
 import { toSimplifiedMessages } from "./messagesTestUtils.js";
+import { createAsyncStorage, createSyncStorage } from "./testStorage.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -301,6 +303,7 @@ export function blockMessageTypeOnOutgoingPeer(
 
   pushSpy.mockImplementation(async (msg) => {
     if (
+      typeof msg === "object" &&
       msg.action === messageType &&
       (!opts.id || msg.id === opts.id) &&
       (!opts.once || !blockedIds.has(msg.id))
@@ -459,31 +462,9 @@ export function getSyncServerConnectedPeer(opts: {
   };
 }
 
-export function createMockStoragePeer(opts: {
-  ourName?: string;
-  peerId: string;
-}) {
-  const storage = createTestNode();
-
-  const { peer1, peer2 } = connectedPeersWithMessagesTracking({
-    peer1: { id: storage.getCurrentAgent().id, role: "storage" },
-    peer2: {
-      id: opts.peerId,
-      role: "client",
-      name: opts.ourName,
-    },
-  });
-
-  peer1.role = "storage";
-  peer1.priority = 100;
-
-  storage.syncManager.addPeer(peer2);
-
-  return {
-    storage,
-    peer: peer1,
-  };
-}
+export const TEST_NODE_CONFIG = {
+  withAsyncPeers: false,
+};
 
 export function setupTestNode(
   opts: {
@@ -521,25 +502,41 @@ export function setupTestNode(
     };
   }
 
-  function addStoragePeer(opts: { ourName?: string } = {}) {
-    const { peer, storage } = createMockStoragePeer({
-      peerId: node.getCurrentAgent().id,
-      ourName: opts.ourName,
-    });
+  function addStorage(opts: { ourName?: string; storage?: StorageAPI } = {}) {
+    const storage =
+      opts.storage ??
+      createSyncStorage({
+        nodeName: opts.ourName ?? "client",
+        storageName: "storage",
+      });
+    node.setStorage(storage);
 
-    node.syncManager.addPeer(peer);
+    return { storage };
+  }
+
+  async function addAsyncStorage(opts: { ourName?: string } = {}) {
+    const storage = await createAsyncStorage({
+      nodeName: opts.ourName ?? "client",
+      storageName: "storage",
+    });
+    node.setStorage(storage);
 
-    return { peer, peerState: node.syncManager.peers[peer.id]!, storage };
+    return { storage };
   }
 
   if (opts.connected) {
     connectToSyncServer();
   }
 
+  onTestFinished(() => {
+    node.gracefulShutdown();
+  });
+
   const ctx = {
     node,
     connectToSyncServer,
-    addStoragePeer,
+    addStorage,
+    addAsyncStorage,
     restart: () => {
       node.gracefulShutdown();
       ctx.node = node = new LocalNode(admin.agentSecret, session, Crypto);
@@ -559,12 +556,14 @@ export async function setupTestAccount(
   opts: {
     isSyncServer?: boolean;
     connected?: boolean;
+    storage?: StorageAPI;
   } = {},
 ) {
   const ctx = await LocalNode.withNewlyCreatedAccount({
     peersToLoadFrom: [],
     crypto: Crypto,
     creationProps: { name: "Client" },
+    storage: opts.storage,
   });
 
   if (opts.isSyncServer) {
@@ -599,26 +598,42 @@ export async function setupTestAccount(
     };
   }
 
-  function addStoragePeer(opts: { ourName?: string } = {}) {
-    const { peer, storage } = createMockStoragePeer({
-      peerId: ctx.node.getCurrentAgent().id,
-      ourName: opts.ourName,
-    });
+  function addStorage(opts: { ourName?: string; storage?: StorageAPI } = {}) {
+    const storage =
+      opts.storage ??
+      createSyncStorage({
+        nodeName: opts.ourName ?? "client",
+        storageName: "storage",
+      });
+    ctx.node.setStorage(storage);
 
-    ctx.node.syncManager.addPeer(peer);
+    return { storage };
+  }
 
-    return { peer, peerState: ctx.node.syncManager.peers[peer.id]!, storage };
+  async function addAsyncStorage(opts: { ourName?: string } = {}) {
+    const storage = await createAsyncStorage({
+      nodeName: opts.ourName ?? "client",
+      storageName: "storage",
+    });
+    ctx.node.setStorage(storage);
+
+    return { storage };
   }
 
   if (opts.connected) {
     connectToSyncServer();
   }
 
+  onTestFinished(() => {
+    ctx.node.gracefulShutdown();
+  });
+
   return {
     node: ctx.node,
     accountID: ctx.accountID,
     connectToSyncServer,
-    addStoragePeer,
+    addStorage,
+    addAsyncStorage,
   };
 }
 
@@ -639,22 +654,42 @@ export function connectedPeersWithMessagesTracking(opts: {
 
   const peer1Push = peer1.outgoing.push;
   peer1.outgoing.push = (msg) => {
-    SyncMessagesLog.add({
-      from: opts.peer2.name ?? opts.peer2.role,
-      to: opts.peer1.name ?? opts.peer1.role,
-      msg,
-    });
-    return peer1Push.call(peer1.outgoing, msg);
+    if (typeof msg !== "string") {
+      SyncMessagesLog.add({
+        from: opts.peer2.name ?? opts.peer2.role,
+        to: opts.peer1.name ?? opts.peer1.role,
+        msg,
+      });
+    }
+
+    if (!TEST_NODE_CONFIG.withAsyncPeers) {
+      peer1Push.call(peer1.outgoing, msg);
+    } else {
+      // Simulate the async nature of the real push
+      setTimeout(() => {
+        peer1Push.call(peer1.outgoing, msg);
+      }, 0);
+    }
   };
 
   const peer2Push = peer2.outgoing.push;
   peer2.outgoing.push = (msg) => {
-    SyncMessagesLog.add({
-      from: opts.peer1.name ?? opts.peer1.role,
-      to: opts.peer2.name ?? opts.peer2.role,
-      msg,
-    });
-    return peer2Push.call(peer2.outgoing, msg);
+    if (typeof msg !== "string") {
+      SyncMessagesLog.add({
+        from: opts.peer1.name ?? opts.peer1.role,
+        to: opts.peer2.name ?? opts.peer2.role,
+        msg,
+      });
+    }
+
+    if (!TEST_NODE_CONFIG.withAsyncPeers) {
+      peer2Push.call(peer2.outgoing, msg);
+    } else {
+      // Simulate the async nature of the real push
+      setTimeout(() => {
+        peer2Push.call(peer2.outgoing, msg);
+      }, 0);
+    }
   };
 
   return {