cojson 0.19.21 → 0.20.0

package/src/tests/sync.deleted.test.ts

@@ -0,0 +1,294 @@
+import { assert, beforeEach, describe, expect, test } from "vitest";
+import { expectMap } from "../coValue";
+import {
+  SyncMessagesLog,
+  TEST_NODE_CONFIG,
+  loadCoValueOrFail,
+  setupTestAccount,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
+import { isDeleteSessionID, SessionID } from "../ids";
+
+let jazzCloud: ReturnType<typeof setupTestNode>;
+
+beforeEach(async () => {
+  // We want to simulate a real world communication that happens asynchronously
+  TEST_NODE_CONFIG.withAsyncPeers = true;
+
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
+describe("syncing deleted coValues", () => {
+  test("client loads a deleted coValue from server (tombstone-only)", async () => {
+    const { node: client } = setupTestNode({ connected: true });
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    // Delete on the server before the client loads.
+    map.core.deleteCoValue();
+    expect(map.core.isDeleted).toBe(true);
+
+    const mapOnClient = await loadCoValueOrFail(client, map.id);
+    const mapCoreOnClient = client.expectCoValueLoaded(map.id);
+
+    expect(mapCoreOnClient.isDeleted).toBe(true);
+    // Historical content should not be synced.
+    expect(mapOnClient.get("hello")).toBeUndefined();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | LOAD Map sessions: empty",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | KNOWN Group sessions: header/3",
+        "client -> server | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("inbound filtering: after deletion, non-delete sessions in the same content message are ignored", async () => {
+    const client = setupTestNode({ connected: false });
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("k", "v", "trusting");
+
+    const contentBeforeDelete = map.core.newContentSince(undefined)?.[0];
+    assert(contentBeforeDelete);
+
+    // Create a delete marker on the server, but also keep a historical session around.
+    map.core.deleteCoValue();
+
+    const content = map.core.newContentSince(undefined)?.[0];
+    assert(content);
+
+    const groupContent = group.core.newContentSince(undefined)?.[0];
+    assert(groupContent);
+
+    // We merge the content before delete with the content after delete to simulate an older peer that might send extra sessions in the same message
+    Object.assign(contentBeforeDelete.new, content.new);
+
+    client.node.syncManager.handleNewContent(groupContent, "import");
+    client.node.syncManager.handleNewContent(content, "import");
+
+    const coreOnClient = client.node.expectCoValueLoaded(map.id);
+    expect(coreOnClient.isDeleted).toBe(true);
+
+    const contentOnClient = expectMap(coreOnClient.getCurrentContent());
+    expect(contentOnClient.get("k")).toBeUndefined();
+  });
+
+  test("should wait for the dependencies to be available before processing the deleted session/transaction", async () => {
+    const client = setupTestNode({ connected: false });
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+
+    // Create a delete marker on the server, but also keep a historical session around.
+    map.core.deleteCoValue();
+
+    const content = map.core.newContentSince(undefined)?.[0];
+    assert(content);
+
+    const groupContent = group.core.newContentSince(undefined)?.[0];
+    assert(groupContent);
+
+    client.node.syncManager.handleNewContent(content, "import");
+    client.node.syncManager.handleNewContent(groupContent, "import");
+
+    await waitFor(() => {
+      expect(client.node.expectCoValueLoaded(map.id).isDeleted).toBe(true);
+    });
+  });
+
+  test("outbound blocking: post-delete normal writes are ignored and do not produce content uploads", async () => {
+    const client = setupTestNode({ connected: true });
+
+    const group = jazzCloud.node.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("a", 1, "trusting");
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+
+    // Delete on the server and wait for it to propagate.
+    map.core.deleteCoValue();
+    await waitFor(() => {
+      expect(mapOnClient.core.isDeleted).toBe(true);
+    });
+
+    SyncMessagesLog.clear();
+
+    mapOnClient.set("x", "y", "trusting");
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    // Ensure we didn't produce outgoing content uploads as a result of the rejected write.
+    const messages = SyncMessagesLog.getMessages({
+      Group: group.core,
+      Map: map.core,
+    });
+    expect(messages.some((m) => m.includes("CONTENT Map"))).toBe(false);
+  });
+
+  test("delete should be propagated to client-to-client sync", async () => {
+    const alice = setupTestNode();
+    alice.connectToSyncServer({
+      ourName: "alice",
+    });
+    const bob = setupTestNode();
+    bob.connectToSyncServer({
+      ourName: "bob",
+    });
+
+    const group = alice.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+    map.core.deleteCoValue();
+
+    await loadCoValueOrFail(bob.node, map.id);
+
+    await waitFor(() => {
+      expect(bob.node.expectCoValueLoaded(map.id).isDeleted).toBe(true);
+    });
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "bob -> server | LOAD Map sessions: empty",
+        "alice -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "alice -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> bob | KNOWN Map sessions: empty",
+        "server -> alice | KNOWN Group sessions: header/3",
+        "server -> alice | KNOWN Map sessions: header/1",
+        "server -> bob | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> bob | CONTENT Map header: true new: After: 0 New: 1",
+        "bob -> server | KNOWN Group sessions: header/3",
+        "bob -> server | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("content synced after deletion should be ignored", async () => {
+    const alice = setupTestNode({ connected: true });
+    const bob = setupTestNode();
+
+    const { peerState: bobConnection } = bob.connectToSyncServer({
+      ourName: "bob",
+    });
+
+    const group = alice.node.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    const mapOnBob = await loadCoValueOrFail(bob.node, map.id);
+
+    SyncMessagesLog.clear();
+
+    bobConnection.gracefulShutdown();
+
+    map.core.deleteCoValue();
+
+    await map.core.waitForSync();
+
+    mapOnBob.set("hello", "updated", "trusting");
+
+    bob.connectToSyncServer({
+      ourName: "bob",
+    });
+
+    await mapOnBob.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> client | KNOWN Map sessions: header/2",
+        "server -> bob | CONTENT Map header: false new: After: 0 New: 1",
+        "bob -> server | LOAD Group sessions: header/5",
+        "bob -> server | LOAD Map sessions: header/2",
+        "bob -> server | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> bob | KNOWN Group sessions: header/5",
+        "server -> bob | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> bob | KNOWN Map sessions: header/3",
+        "bob -> server | KNOWN Map sessions: header/2",
+      ]
+    `);
+  });
+
+  test("should handle concurrent delete operations", async () => {
+    const alice = await setupTestAccount();
+    alice.connectToSyncServer({
+      ourName: "alice",
+    });
+    const bob = await setupTestAccount();
+    bob.connectToSyncServer({
+      ourName: "bob",
+    });
+
+    const group = jazzCloud.node.createGroup();
+    group.addMemberInternal(alice.account, "admin");
+    group.addMemberInternal(bob.account, "admin");
+
+    const map = group.createMap();
+    map.set("counter", 0, "trusting");
+    map.set("counter", 1, "trusting");
+
+    const mapOnAlice = await loadCoValueOrFail(alice.node, map.id);
+    const mapOnBob = await loadCoValueOrFail(bob.node, map.id);
+
+    SyncMessagesLog.clear();
+
+    mapOnAlice.core.deleteCoValue();
+    mapOnBob.core.deleteCoValue();
+
+    await mapOnAlice.core.waitForSync();
+    await mapOnBob.core.waitForSync();
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "alice -> server | CONTENT Map header: false new: After: 0 New: 1",
+        "bob -> server | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> alice | KNOWN Map sessions: header/3",
+        "server -> bob | CONTENT Map header: false new: After: 0 New: 1",
+        "server -> bob | KNOWN Map sessions: header/4",
+        "server -> alice | CONTENT Map header: false new: After: 0 New: 1",
+        "bob -> server | KNOWN Map sessions: header/4",
+      ]
+    `);
+
+    expect(map.core.isDeleted).toBe(true);
+
+    const sessions = map.core.knownState().sessions;
+
+    expect(Object.keys(sessions)).toHaveLength(2);
+    expect(
+      Object.keys(sessions).every((sessionID) =>
+        isDeleteSessionID(sessionID as SessionID),
+      ),
+    ).toBe(true);
+  });
+});

package/src/tests/sync.garbageCollection.test.ts

@@ -50,6 +50,47 @@ describe("sync after the garbage collector has run", () => {
     const mapOnClient = await loadCoValueOrFail(client.node, map.id);
     expect(mapOnClient.get("hello")).toEqual("world");
 
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | LOAD Map sessions: empty",
+        "server -> storage | LOAD Map sessions: empty",
+        "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | KNOWN Group sessions: header/3",
+        "client -> server | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("loading a coValue from the sync server that was removed by the garbage collector along with its owner", async () => {
+    const client = setupTestNode();
+
+    client.connectToSyncServer();
+
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+
+    // force the garbage collector to run twice to remove the map and its group
+    jazzCloud.node.garbageCollector?.collect();
+    jazzCloud.node.garbageCollector?.collect();
+
+    expect(jazzCloud.node.getCoValue(group.id).isAvailable()).toBe(false);
+    expect(jazzCloud.node.getCoValue(map.id).isAvailable()).toBe(false);
+
+    SyncMessagesLog.clear();
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+
     expect(
       SyncMessagesLog.getMessages({
         Group: group.core,
@@ -103,7 +144,6 @@ describe("sync after the garbage collector has run", () => {
       [
         "client -> server | CONTENT Map header: false new: After: 0 New: 1",
         "server -> storage | LOAD Map sessions: empty",
-        "storage -> server | CONTENT Group header: true new: After: 0 New: 5",
         "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
         "server -> client | KNOWN Map sessions: header/2",
         "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
@@ -112,41 +152,30 @@ describe("sync after the garbage collector has run", () => {
   });
 
   test("syncing a coValue that was removed by the garbage collector", async () => {
-    const edge = setupTestNode();
-    edge.addStorage({
-      ourName: "edge",
-    });
-    edge.connectToSyncServer({
-      syncServer: jazzCloud.node,
-      syncServerName: "server",
-      ourName: "edge",
-    });
-    edge.node.enableGarbageCollector();
     const client = setupTestNode();
-
-    client.connectToSyncServer({
-      syncServer: edge.node,
-      syncServerName: "edge",
+    client.addStorage({
+      ourName: "client",
     });
+    client.node.enableGarbageCollector();
 
-    const group = edge.node.createGroup();
-    group.addMember("everyone", "writer");
-
-    await group.core.waitForSync();
-
+    const group = client.node.createGroup();
     const map = group.createMap();
-
     map.set("hello", "updated", "trusting");
 
     // force the garbage collector to run before the transaction is synced
-    edge.node.garbageCollector?.collect();
-    expect(edge.node.getCoValue(map.id).isAvailable()).toBe(false);
+    client.node.garbageCollector?.collect();
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
 
     SyncMessagesLog.clear();
 
+    client.connectToSyncServer();
+
+    // Wait for unsynced coValues to be resumed and synced after connecting to server
+    await client.node.syncManager.waitForAllCoValuesSync();
+
     // The storage should work even after the coValue is unmounted, so the load should be successful
-    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
-    expect(mapOnClient.get("hello")).toEqual("updated");
+    const mapOnServer = await loadCoValueOrFail(jazzCloud.node, map.id);
+    expect(mapOnServer.get("hello")).toEqual("updated");
 
     expect(
       SyncMessagesLog.getMessages({
@@ -155,18 +184,22 @@ describe("sync after the garbage collector has run", () => {
       }),
     ).toMatchInlineSnapshot(`
       [
-        "client -> edge | LOAD Map sessions: empty",
-        "edge -> storage | CONTENT Map header: true new: After: 0 New: 1",
-        "edge -> server | CONTENT Map header: true new: After: 0 New: 1",
-        "edge -> storage | LOAD Map sessions: empty",
-        "storage -> edge | CONTENT Group header: true new: After: 0 New: 5",
-        "storage -> edge | CONTENT Map header: true new: After: 0 New: 1",
-        "edge -> client | CONTENT Group header: true new: After: 0 New: 5",
-        "edge -> client | CONTENT Map header: true new: After: 0 New: 1",
-        "server -> edge | KNOWN Map sessions: header/1",
+        "client -> server | LOAD Map sessions: empty",
+        "client -> server | LOAD Group sessions: header/3",
+        "client -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> storage | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> storage | LOAD Map sessions: empty",
+        "storage -> server | KNOWN Map sessions: empty",
+        "server -> client | KNOWN Map sessions: empty",
+        "server -> storage | GET_KNOWN_STATE Group",
+        "storage -> server | GET_KNOWN_STATE_RESULT Group sessions: empty",
+        "server -> client | KNOWN Group sessions: empty",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> storage | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN Map sessions: header/1",
         "server -> storage | CONTENT Map header: true new: After: 0 New: 1",
-        "client -> edge | KNOWN Group sessions: header/5",
-        "client -> edge | KNOWN Map sessions: header/1",
       ]
     `);
   });

package/src/tests/sync.load.test.ts

@@ -514,7 +514,7 @@ describe("loading coValues from server", () => {
     });
 
     // Makes the CoValues unavailable on the server
-    jazzCloud.restart();
+    await jazzCloud.restart();
 
     const client = setupTestNode({
       connected: true,
@@ -1476,7 +1476,7 @@ describe("lazy storage load optimization", () => {
 
     // Restart the server to clear memory (keeping storage)
     // Now the server has no CoValues in memory, only in storage
-    jazzCloud.restart();
+    await jazzCloud.restart();
     jazzCloud.node.setStorage(storage);
 
     SyncMessagesLog.clear();
@@ -1519,7 +1519,7 @@ describe("lazy storage load optimization", () => {
     await map.core.waitForSync();
 
     // Restart the server to clear memory (keeping storage)
-    jazzCloud.restart();
+    await jazzCloud.restart();
     jazzCloud.node.setStorage(storage);
 
     SyncMessagesLog.clear();
@@ -1643,7 +1643,6 @@ describe("lazy storage load optimization", () => {
       [
         "client -> server | CONTENT Map header: false new: After: 0 New: 1",
         "server -> storage | LOAD Map sessions: empty",
-        "storage -> server | CONTENT Group header: true new: After: 0 New: 5",
         "storage -> server | CONTENT Map header: true new: After: 0 New: 1",
         "server -> client | KNOWN Map sessions: header/2",
         "server -> storage | CONTENT Map header: false new: After: 0 New: 1",
@@ -1702,7 +1701,6 @@ describe("lazy storage load optimization", () => {
       [
         "client -> server | CONTENT Map header: false new: After: 0 New: 1",
         "server -> storage | LOAD Map sessions: empty",
-        "storage -> server | CONTENT Group header: true new: After: 0 New: 5",
         "storage -> server | CONTENT Map header: true new: After: 0 New: 73 expectContentUntil: header/201",
         "server -> client | KNOWN Map sessions: header/74",
         "server -> storage | CONTENT Map header: false new: After: 0 New: 1",

package/src/tests/sync.mesh.test.ts

@@ -11,7 +11,8 @@ import {
   setupTestNode,
   waitFor,
 } from "./testUtils";
-import { stableStringify } from "../jsonStringify";
+import { Stringified } from "../jsonStringify";
+import { JsonValue } from "../jsonValue";
 
 // We want to simulate a real world communication that happens asynchronously
 TEST_NODE_CONFIG.withAsyncPeers = true;
@@ -325,7 +326,9 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
 
     msg.new[mesh.edgeFrance.node.currentSessionID]!.newTransactions.push({
       privacy: "trusting",
-      changes: stableStringify([{ op: "set", key: "hello", value: "updated" }]),
+      changes: JSON.stringify([
+        { op: "set", key: "hello", value: "updated" },
+      ]) as Stringified<JsonValue[]>,
       madeAt: Date.now(),
     });
 
@@ -506,7 +509,7 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
       ]
     `);
 
-    edge.restart();
+    await edge.restart();
 
     edge.connectToSyncServer({
       syncServerName: "core",

package/src/tests/sync.peerReconciliation.test.ts

@@ -167,7 +167,7 @@ describe("peer reconciliation", () => {
 
     await map.core.waitForSync();
 
-    jazzCloud.restart();
+    await jazzCloud.restart();
     SyncMessagesLog.clear();
     client.connectToSyncServer();
 
@@ -222,7 +222,7 @@ describe("peer reconciliation", () => {
 
     await map.core.waitForSync();
 
-    jazzCloud.restart();
+    await jazzCloud.restart();
     SyncMessagesLog.clear();
     client.connectToSyncServer();
 
@@ -305,7 +305,7 @@ describe("peer reconciliation", () => {
 
     await map.core.waitForSync();
 
-    jazzCloud.restart();
+    await jazzCloud.restart();
 
     SyncMessagesLog.clear();
     client.connectToSyncServer();

package/src/tests/sync.storage.test.ts

@@ -8,6 +8,7 @@ import {
   vi,
 } from "vitest";
 
+import type { JsonValue } from "../exports";
 import { cojsonInternals, emptyKnownState } from "../exports";
 import {
   SyncMessagesLog,
@@ -19,7 +20,7 @@ import {
   tearDownTestMetricReader,
   waitFor,
 } from "./testUtils";
-import { stableStringify } from "../jsonStringify";
+import { Stringified } from "../jsonStringify";
 
 // We want to simulate a real world communication that happens asynchronously
 TEST_NODE_CONFIG.withAsyncPeers = true;
@@ -83,7 +84,7 @@ describe("client with storage syncs with server", () => {
 
     await loadCoValueOrFail(client.node, map.id);
 
-    client.restart();
+    await client.restart();
 
     client.connectToSyncServer();
     client.addStorage({
@@ -167,7 +168,7 @@ describe("client with storage syncs with server", () => {
 
     await map.core.waitForSync();
 
-    client.restart();
+    await client.restart();
 
     client.addStorage({
       storage,
@@ -217,7 +218,7 @@ describe("client with storage syncs with server", () => {
     branch.set("branchKey", "branchValue");
     await branch.core.waitForSync();
 
-    client.restart();
+    await client.restart();
     client.addStorage({
       storage,
     });
@@ -388,7 +389,7 @@ describe("client syncs with a server with storage", () => {
 
     SyncMessagesLog.clear();
 
-    client.restart();
+    await client.restart();
 
     client.connectToSyncServer({
       ourName: "client",
@@ -457,7 +458,7 @@ describe("client syncs with a server with storage", () => {
 
     expect(correctionSpy).not.toHaveBeenCalled();
 
-    client.restart();
+    await client.restart();
 
     client.connectToSyncServer({
       ourName: "client",
@@ -572,7 +573,9 @@ describe("client syncs with a server with storage", () => {
     const invalidMapContent = structuredClone(mapContent);
     invalidMapContent.new[bob.node.currentSessionID]!.newTransactions.push({
       privacy: "trusting",
-      changes: stableStringify([{ op: "set", key: "hello", value: "updated" }]),
+      changes: JSON.stringify([
+        { op: "set", key: "hello", value: "updated" },
+      ]) as Stringified<JsonValue[]>,
       madeAt: Date.now(),
     });
     client.node.syncManager.handleNewContent(invalidMapContent, "import");
@@ -771,7 +774,7 @@ describe("client syncs with a server with storage", () => {
 
     SyncMessagesLog.clear();
 
-    syncServer.restart();
+    await syncServer.restart();
     syncServer.addStorage({
       ourName: "syncServer",
       storage,
@@ -848,7 +851,7 @@ describe("client syncs with a server with storage", () => {
       ]);
 
       // Restart to load from storage
-      client.restart();
+      await client.restart();
       client.addStorage({ storage });
 
       // Load all maps concurrently from storage
@@ -892,7 +895,7 @@ describe("client syncs with a server with storage", () => {
       SyncMessagesLog.clear();
 
       // Restart client with storage
-      client.restart();
+      await client.restart();
       client.connectToSyncServer();
       client.addStorage({ storage });
 
@@ -985,7 +988,7 @@ describe("client syncs with a server with storage", () => {
 
       SyncMessagesLog.clear();
 
-      syncServer.restart();
+      await syncServer.restart();
       syncServer.addStorage({
         ourName: "syncServer",
         storage,