cojson-storage-indexeddb 0.19.17 → 0.19.19

package/src/idbNode.ts

@@ -9,7 +9,7 @@ export function internal_setDatabaseName(name: string) {
 
 export async function getIndexedDBStorage(name = DATABASE_NAME) {
   const dbPromise = new Promise<IDBDatabase>((resolve, reject) => {
-    const request = indexedDB.open(name, 4);
+    const request = indexedDB.open(name, 5);
     request.onerror = () => {
       reject(request.error);
     };
@@ -47,6 +47,20 @@ export async function getIndexedDBStorage(name = DATABASE_NAME) {
           keyPath: ["ses", "idx"],
         });
       }
+      if (ev.oldVersion <= 4) {
+        const unsyncedCoValues = db.createObjectStore("unsyncedCoValues", {
+          autoIncrement: true,
+          keyPath: "rowID",
+        });
+        unsyncedCoValues.createIndex("byCoValueId", "coValueId");
+        unsyncedCoValues.createIndex(
+          "uniqueUnsyncedCoValues",
+          ["coValueId", "peerId"],
+          {
+            unique: true,
+          },
+        );
+      }
     };
   });
 

package/src/tests/CoJsonIDBTransaction.test.ts

@@ -23,6 +23,17 @@ describe("CoJsonIDBTransaction", () => {
         });
         db.createObjectStore("transactions", { keyPath: "id" });
         db.createObjectStore("signatureAfter", { keyPath: "id" });
+        const unsyncedCoValues = db.createObjectStore("unsyncedCoValues", {
+          keyPath: "rowID",
+        });
+        unsyncedCoValues.createIndex("byCoValueId", "coValueId");
+        unsyncedCoValues.createIndex(
+          "uniqueUnsyncedCoValues",
+          ["coValueId", "peerId"],
+          {
+            unique: true,
+          },
+        );
       };
 
       request.onsuccess = () => {
@@ -167,4 +178,80 @@ describe("CoJsonIDBTransaction", () => {
 
     expect(badTx.failed).toBe(true);
   });
+
+  test("transaction with custom stores only includes specified stores", async () => {
+    const tx = new CoJsonIDBTransaction(db, ["coValues", "sessions"]);
+
+    // Should work with included stores
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("coValues").put({
+        id: "test1",
+        value: "hello",
+      }),
+    );
+
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("sessions").put({
+        id: "session1",
+        data: "session data",
+      }),
+    );
+
+    // Should fail when trying to access a store not included in transaction
+    await expect(
+      tx.handleRequest((tx) =>
+        tx.getObjectStore("transactions").put({
+          id: "tx1",
+          data: "tx data",
+        }),
+      ),
+    ).rejects.toThrow(
+      "Failed to execute 'objectStore' on 'IDBTransaction': The specified object store was not found.",
+    );
+  });
+
+  test("if no custom stores are provided, transaction uses default stores", async () => {
+    const tx = new CoJsonIDBTransaction(db);
+
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("coValues").put({
+        id: "test1",
+        value: "hello",
+      }),
+    );
+
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("sessions").put({
+        id: "session1",
+        data: "session data",
+      }),
+    );
+
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("transactions").put({
+        id: "tx1",
+        data: "tx data",
+      }),
+    );
+
+    await tx.handleRequest((tx) =>
+      tx.getObjectStore("signatureAfter").put({
+        id: "sig1",
+        data: "sig data",
+      }),
+    );
+
+    // Should fail when trying to access unsyncedCoValues (not in default)
+    await expect(
+      tx.handleRequest((tx) =>
+        tx.getObjectStore("unsyncedCoValues").put({
+          rowID: 1,
+          coValueId: "coValue1",
+          peerId: "peer1",
+        }),
+      ),
+    ).rejects.toThrow(
+      "Failed to execute 'objectStore' on 'IDBTransaction': The specified object store was not found.",
+    );
+  });
 });

package/src/tests/storage.indexeddb.test.ts

@@ -1,29 +1,37 @@
-import { LocalNode, StorageApiAsync } from "cojson";
+import { LocalNode, StorageApiAsync, cojsonInternals } from "cojson";
 import { WasmCrypto } from "cojson/crypto/WasmCrypto";
-import { afterEach, beforeEach, expect, test, vi } from "vitest";
-import { getIndexedDBStorage } from "../index.js";
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { getIndexedDBStorage, internal_setDatabaseName } from "../index.js";
 import { toSimplifiedMessages } from "./messagesTestUtils.js";
-import { trackMessages, waitFor } from "./testUtils.js";
+import {
+  clearObjectStore,
+  connectToSyncServer,
+  createTestNode,
+  trackMessages,
+  waitFor,
+} from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 let syncMessages: ReturnType<typeof trackMessages>;
 
+const DATABASE_NAME = "jazz-storage";
+internal_setDatabaseName(DATABASE_NAME);
+
 beforeEach(() => {
   syncMessages = trackMessages();
+  cojsonInternals.setSyncStateTrackingBatchDelay(0);
+  cojsonInternals.setCoValueLoadingRetryDelay(10);
 });
 
-afterEach(() => {
+afterEach(async () => {
   syncMessages.restore();
+  cojsonInternals.setSyncStateTrackingBatchDelay(1000);
+
+  await clearObjectStore(DATABASE_NAME, "unsyncedCoValues");
 });
 
 test("should sync and load data from storage", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
   node1.setStorage(await getIndexedDBStorage());
 
   const group = node1.createGroup();
@@ -51,12 +59,7 @@ test("should sync and load data from storage", async () => {
   node1.gracefulShutdown();
   syncMessages.clear();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
-
+  const node2 = createTestNode({ secret: node1.agentSecret });
   node2.setStorage(await getIndexedDBStorage());
 
   const map2 = await node2.load(map.id);
@@ -84,13 +87,7 @@ test("should sync and load data from storage", async () => {
 });
 
 test("should send an empty content message if there is no content", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   node1.setStorage(await getIndexedDBStorage());
 
@@ -117,11 +114,7 @@ test("should send an empty content message if there is no content", async () =>
   syncMessages.clear();
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   node2.setStorage(await getIndexedDBStorage());
 
@@ -148,13 +141,7 @@ test("should send an empty content message if there is no content", async () =>
 });
 
 test("should load dependencies correctly (group inheritance)", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   node1.setStorage(await getIndexedDBStorage());
   const group = node1.createGroup();
@@ -189,11 +176,7 @@ test("should load dependencies correctly (group inheritance)", async () => {
   syncMessages.clear();
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   node2.setStorage(await getIndexedDBStorage());
 
@@ -223,13 +206,7 @@ test("should load dependencies correctly (group inheritance)", async () => {
 });
 
 test("should not send the same dependency value twice", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   node1.setStorage(await getIndexedDBStorage());
 
@@ -250,11 +227,7 @@ test("should not send the same dependency value twice", async () => {
   syncMessages.clear();
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   node2.setStorage(await getIndexedDBStorage());
 
@@ -289,13 +262,7 @@ test("should not send the same dependency value twice", async () => {
 });
 
 test("should recover from data loss", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   const storage = await getIndexedDBStorage();
   node1.setStorage(storage);
@@ -347,11 +314,7 @@ test("should recover from data loss", async () => {
   syncMessages.clear();
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   node2.setStorage(await getIndexedDBStorage());
 
@@ -386,13 +349,7 @@ test("should recover from data loss", async () => {
 });
 
 test("should sync multiple sessions in a single content message", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   node1.setStorage(await getIndexedDBStorage());
 
@@ -406,11 +363,7 @@ test("should sync multiple sessions in a single content message", async () => {
 
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   node2.setStorage(await getIndexedDBStorage());
 
@@ -427,11 +380,7 @@ test("should sync multiple sessions in a single content message", async () => {
 
   node2.gracefulShutdown();
 
-  const node3 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node3 = createTestNode({ secret: node1.agentSecret });
 
   syncMessages.clear();
 
@@ -462,13 +411,7 @@ test("should sync multiple sessions in a single content message", async () => {
 });
 
 test("large coValue upload streaming", async () => {
-  const agentSecret = Crypto.newRandomAgentSecret();
-
-  const node1 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node1 = createTestNode();
 
   node1.setStorage(await getIndexedDBStorage());
 
@@ -494,11 +437,7 @@ test("large coValue upload streaming", async () => {
 
   node1.gracefulShutdown();
 
-  const node2 = new LocalNode(
-    agentSecret,
-    Crypto.newRandomSessionID(Crypto.getAgentID(agentSecret)),
-    Crypto,
-  );
+  const node2 = createTestNode({ secret: node1.agentSecret });
 
   syncMessages.clear();
 
@@ -603,3 +542,115 @@ test("should sync and load accounts from storage", async () => {
 
   expect(node2.getCoValue(accountID).isAvailable()).toBeTruthy();
 });
+
+describe("sync state persistence", () => {
+  test("unsynced coValues are asynchronously persisted to storage", async () => {
+    // Client is not connected to a sync server, so sync will not be completed
+    const client = createTestNode();
+    client.setStorage(await getIndexedDBStorage());
+
+    const group = client.createGroup();
+    const map = group.createMap();
+    map.set("key", "value");
+
+    // Wait for the unsynced coValues to be persisted to storage
+    await new Promise<void>((resolve) => setTimeout(resolve, 500));
+
+    const unsyncedCoValueIDs = await new Promise((resolve) =>
+      client.storage?.getUnsyncedCoValueIDs(resolve),
+    );
+    expect(unsyncedCoValueIDs).toHaveLength(2);
+    expect(unsyncedCoValueIDs).toContain(map.id);
+    expect(unsyncedCoValueIDs).toContain(group.id);
+
+    await client.gracefulShutdown();
+  });
+
+  test("synced coValues are removed from storage", async () => {
+    const syncServer = createTestNode();
+    const client = createTestNode();
+    client.setStorage(await getIndexedDBStorage());
+
+    connectToSyncServer(client, syncServer);
+
+    const group = client.createGroup();
+    const map = group.createMap();
+    map.set("key", "value");
+
+    // Wait enough time for the coValue to be synced
+    await new Promise<void>((resolve) => setTimeout(resolve, 500));
+
+    const unsyncedCoValueIDs = await new Promise((resolve) =>
+      client.storage?.getUnsyncedCoValueIDs(resolve),
+    );
+    expect(unsyncedCoValueIDs).toHaveLength(0);
+    expect(client.syncManager.unsyncedTracker.has(map.id)).toBe(false);
+
+    await client.gracefulShutdown();
+    await syncServer.gracefulShutdown();
+  });
+
+  test("unsynced coValues are persisted to storage when the node is shutdown", async () => {
+    const client = createTestNode();
+    client.setStorage(await getIndexedDBStorage());
+
+    const group = client.createGroup();
+    const map = group.createMap();
+    map.set("key", "value");
+
+    // Wait for local transaction to trigger sync
+    await new Promise<void>((resolve) => queueMicrotask(resolve));
+
+    await client.gracefulShutdown();
+
+    const unsyncedCoValueIDs = await new Promise((resolve) =>
+      client.storage?.getUnsyncedCoValueIDs(resolve),
+    );
+    expect(unsyncedCoValueIDs).toHaveLength(2);
+    expect(unsyncedCoValueIDs).toContain(map.id);
+    expect(unsyncedCoValueIDs).toContain(group.id);
+  });
+});
+
+describe("sync resumption", () => {
+  test("unsynced coValues are resumed when the node is restarted", async () => {
+    // Client is not connected to a sync server, so sync will not be completed
+    const node1 = createTestNode();
+    const storage = await getIndexedDBStorage();
+    node1.setStorage(storage);
+
+    const getUnsyncedCoValueIDsFromStorage = async () =>
+      new Promise<string[]>((resolve) =>
+        node1.storage?.getUnsyncedCoValueIDs(resolve),
+      );
+
+    const group = node1.createGroup();
+    const map = group.createMap();
+    map.set("key", "value");
+
+    // Wait for the unsynced coValues to be persisted to storage
+    await new Promise<void>((resolve) => setTimeout(resolve, 100));
+
+    const unsyncedTracker = node1.syncManager.unsyncedTracker;
+    expect(unsyncedTracker.has(map.id)).toBe(true);
+    expect(await getUnsyncedCoValueIDsFromStorage()).toHaveLength(2);
+
+    node1.gracefulShutdown();
+
+    // Create second node with the same storage
+    const node2 = createTestNode();
+    node2.setStorage(storage);
+
+    // Connect to sync server
+    const syncServer = createTestNode();
+    connectToSyncServer(node2, syncServer);
+
+    await node2.syncManager.waitForAllCoValuesSync();
+    // Wait for sync to resume & complete
+    await waitFor(
+      async () => (await getUnsyncedCoValueIDsFromStorage()).length === 0,
+    );
+
+    await node2.gracefulShutdown();
+  });
+});

package/src/tests/testUtils.ts

@@ -1,5 +1,11 @@
-import type { RawCoID, SyncMessage } from "cojson";
-import { StorageApiAsync } from "cojson";
+import type { AgentSecret, RawCoID, SessionID, SyncMessage } from "cojson";
+import {
+  cojsonInternals,
+  ControlledAgent,
+  LocalNode,
+  StorageApiAsync,
+} from "cojson";
+import { WasmCrypto } from "cojson/crypto/WasmCrypto";
 import { onTestFinished } from "vitest";
 
 export function trackMessages() {
@@ -115,3 +121,78 @@ export function waitFor(
     }, 100);
   });
 }
+
+export async function clearObjectStore(
+  dbName: string,
+  storeName: string,
+): Promise<void> {
+  return new Promise((resolve, reject) => {
+    const openReq = indexedDB.open(dbName);
+
+    openReq.onerror = () => reject(openReq.error);
+
+    openReq.onsuccess = () => {
+      const db = openReq.result;
+
+      if (!db.objectStoreNames.contains(storeName)) {
+        db.close();
+        resolve();
+        return;
+      }
+
+      const tx = db.transaction(storeName, "readwrite");
+      const store = tx.objectStore(storeName);
+
+      const clearReq = store.clear();
+
+      clearReq.onerror = () => reject(clearReq.error);
+
+      tx.oncomplete = () => {
+        db.close();
+        resolve();
+      };
+
+      tx.onerror = () => {
+        db.close();
+        reject(tx.error);
+      };
+
+      tx.onabort = () => {
+        db.close();
+        reject(tx.error || new Error("Transaction aborted"));
+      };
+    };
+  });
+}
+
+const Crypto = await WasmCrypto.create();
+
+export function getAgentAndSessionID(
+  secret: AgentSecret = Crypto.newRandomAgentSecret(),
+): [ControlledAgent, SessionID] {
+  const sessionID = Crypto.newRandomSessionID(Crypto.getAgentID(secret));
+  return [new ControlledAgent(secret, Crypto), sessionID];
+}
+
+export function createTestNode(opts?: { secret?: AgentSecret }) {
+  const [admin, session] = getAgentAndSessionID(opts?.secret);
+  return new LocalNode(admin.agentSecret, session, Crypto);
+}
+
+export function connectToSyncServer(
+  client: LocalNode,
+  syncServer: LocalNode,
+): void {
+  const [clientPeer, serverPeer] = cojsonInternals.connectedPeers(
+    client.currentSessionID,
+    syncServer.currentSessionID,
+    {
+      peer1role: "client",
+      peer2role: "server",
+      persistent: true,
+    },
+  );
+
+  client.syncManager.addPeer(serverPeer);
+  syncServer.syncManager.addPeer(clientPeer);
+}

package/vitest.config.ts

@@ -7,7 +7,12 @@ export default defineProject({
     browser: {
       enabled: true,
       provider: playwright(),
-      instances: [{ browser: "chromium", headless: true }],
+      instances: [
+        {
+          headless: process.env.HEADLESS !== "false",
+          browser: "chromium",
+        },
+      ],
     },
     include: ["src/**/*.test.ts"],
   },