cojson 0.19.21 → 0.20.0

package/src/sync.ts

@@ -14,7 +14,7 @@ import {
 import { CoValueCore } from "./coValueCore/coValueCore.js";
 import { CoValueHeader, Transaction } from "./coValueCore/verifiedState.js";
 import { Signature } from "./crypto/crypto.js";
-import { RawCoID, SessionID, isRawCoID } from "./ids.js";
+import { isDeleteSessionID, RawCoID, SessionID, isRawCoID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
 import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
@@ -102,6 +102,10 @@ export interface Peer {
   persistent?: boolean;
 }
 
+function isPersistentServerPeer(peer: Peer | PeerState): boolean {
+  return peer.role === "server" && (peer.persistent ?? false);
+}
+
 export type ServerPeerSelector = (
   id: RawCoID,
   serverPeers: PeerState[],
@@ -209,7 +213,6 @@ export class SyncManager {
       return;
     }
 
-    // TODO: validate
     switch (msg.action) {
       case "load":
         return this.handleLoad(msg, peer);
@@ -261,10 +264,18 @@ export class SyncManager {
 
       peer.combineOptimisticWith(id, coValue.knownState());
     } else if (!peer.toldKnownState.has(id)) {
-      this.trySendToPeer(peer, {
-        action: "known",
-        ...coValue.knownStateWithStreaming(),
-      });
+      if (coValue.isDeleted) {
+        // This way we make the peer believe that we've always ingested all the content they sent, even though we skipped it because the coValue is deleted
+        this.trySendToPeer(
+          peer,
+          coValue.stopSyncingKnownStateMessage(peer.getKnownState(id)),
+        );
+      } else {
+        this.trySendToPeer(peer, {
+          action: "known",
+          ...coValue.knownStateWithStreaming(),
+        });
+      }
     }
 
     peer.trackToldKnownState(id);
@@ -354,7 +365,7 @@ export class SyncManager {
   }
 
   startPeerReconciliation(peer: PeerState) {
-    if (peer.role === "server" && peer.persistent) {
+    if (isPersistentServerPeer(peer)) {
       // Resume syncing unsynced CoValues asynchronously
       this.resumeUnsyncedCoValues().catch((error) => {
         logger.warn("Failed to resume unsynced CoValues:", error);
@@ -525,7 +536,7 @@ export class SyncManager {
 
     const unsubscribeFromKnownStatesUpdates =
       peerState.subscribeToKnownStatesUpdates((id, knownState) => {
-        this.syncState.triggerUpdate(peer.id, id, knownState.value());
+        this.syncState.triggerUpdate(peer, id, knownState.value());
       });
 
     if (!skipReconciliation && peerState.role === "server") {
@@ -711,8 +722,8 @@ export class SyncManager {
 
     peer.combineWith(msg.id, knownStateFrom(msg));
 
-    // The header is a boolean value that tells us if the other peer do have information about the header.
-    // If it's false in this point it means that the coValue is unavailable on the other peer.
+    // The header is a boolean value that tells us if the other peer has information about the header.
+    // If it's false at this point it means that the coValue is unavailable on the other peer.
     const availableOnPeer = peer.getOptimisticKnownState(msg.id)?.header;
 
     if (!availableOnPeer) {
@@ -871,6 +882,8 @@ export class SyncManager {
       new: {},
     };
 
+    let wasAlreadyDeleted = coValue.isDeleted;
+
     /**
      * The coValue is in memory, load the transactions from the content message
      */
@@ -878,6 +891,10 @@ export class SyncManager {
       sessionID,
       newContentForSession,
     ] of getSessionEntriesFromContentMessage(msg)) {
+      if (wasAlreadyDeleted && !isDeleteSessionID(sessionID)) {
+        continue;
+      }
+
       const newTransactions = getNewTransactionsFromContentMessage(
         newContentForSession,
         coValue.knownState(),
@@ -932,12 +949,25 @@ export class SyncManager {
         this.recordTransactionsSize(newTransactions, sourceRole);
       }
 
+      // We reset the new content for the deleted coValue
+      // because we want to store only the delete session/transaction
+      if (!wasAlreadyDeleted && coValue.isDeleted) {
+        wasAlreadyDeleted = true;
+        validNewContent.new = {};
+      }
+
       // The new content for this session has been verified, so we can store it
       validNewContent.new[sessionID] = newContentForSession;
     }
 
     if (peer) {
-      peer.combineWith(msg.id, knownStateFromContent(validNewContent));
+      if (coValue.isDeleted) {
+        // In case of deleted coValues, we combine the known state with the content message
+        // to avoid that clients that don't support deleted coValues try to sync their own content indefinitely
+        peer.combineWith(msg.id, knownStateFromContent(msg));
+      } else {
+        peer.combineWith(msg.id, knownStateFromContent(validNewContent));
+      }
     }
 
     /**
@@ -969,10 +999,18 @@ export class SyncManager {
        * This way the sender knows that the content has been received and applied
        * and can update their peer's knownState accordingly.
        */
-      this.trySendToPeer(peer, {
-        action: "known",
-        ...coValue.knownState(),
-      });
+      if (coValue.isDeleted) {
+        // This way we make the peer believe that we've ingested all the content, even though we skipped it because the coValue is deleted
+        this.trySendToPeer(
+          peer,
+          coValue.stopSyncingKnownStateMessage(peer.getKnownState(msg.id)),
+        );
+      } else {
+        this.trySendToPeer(peer, {
+          action: "known",
+          ...coValue.knownState(),
+        });
+      }
       peer.trackToldKnownState(msg.id);
     }
 
@@ -1076,6 +1114,18 @@ export class SyncManager {
     const isSyncRequired = this.local.syncWhen !== "never";
     if (isSyncRequired && peers.length === 0) {
       this.unsyncedTracker.add(coValueId);
+
+      // Mark CoValue as synced once a persistent server peer is added and
+      // the CoValue is synced
+      const unsubscribe = this.syncState.subscribeToCoValueUpdates(
+        coValueId,
+        (peer, _knownState, syncState) => {
+          if (isPersistentServerPeer(peer) && syncState.uploaded) {
+            this.unsyncedTracker.remove(coValueId);
+            unsubscribe();
+          }
+        },
+      );
       return;
     }
 
@@ -1108,6 +1158,12 @@ export class SyncManager {
 
     const value = this.local.getCoValue(content.id);
 
+    if (value.isDeleted) {
+      // This doesn't persist the delete flag, it only signals the storage
+      // API that the delete transaction is valid
+      storage.markDeleteAsValid(value.id);
+    }
+
     // Try to store the content as-is for performance
     // In case that some transactions are missing, a correction will be requested, but it's an edge case
     storage.store(content, (correction) => {
@@ -1128,6 +1184,19 @@ export class SyncManager {
     });
   }
 
+  /**
+   * Returns true if the local CoValue changes have been synced to all persistent server peers.
+   *
+   * Used during garbage collection to determine if the coValue is pending sync.
+   */
+  isSyncedToServerPeers(id: RawCoID): boolean {
+    // If there are currently no server peers, go ahead with GC.
+    // The CoValue will be reloaded into memory and synced when a peer is added.
+    return this.getPersistentServerPeers(id).every((peer) =>
+      this.syncState.isSynced(peer, id),
+    );
+  }
+
   waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const peerState = this.peers[peerId];
 

package/src/tests/CojsonMessageChannel.test.ts

@@ -0,0 +1,306 @@
+import { describe, test, expect, beforeEach, assert } from "vitest";
+import { CojsonMessageChannel } from "../CojsonMessageChannel";
+import type { Peer } from "../sync.js";
+import type { RawCoMap } from "../coValues/coMap.js";
+import {
+  setupTestNode,
+  SyncMessagesLog,
+  waitFor,
+  createTrackedMessageChannel,
+  createMockWorkerWithAccept,
+  loadCoValueOrFail,
+} from "./testUtils";
+
+describe("CojsonMessageChannel", () => {
+  beforeEach(() => {
+    SyncMessagesLog.clear();
+  });
+
+  test("should sync data between two contexts via MessageChannel", async () => {
+    // Create two nodes using setupTestNode (handles cleanup automatically)
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      // This runs in the "worker" context
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    // Host side: expose to the mock worker
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+      messageChannel: createTrackedMessageChannel({
+        port1Name: "client",
+        port2Name: "server",
+      }),
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1
+    const group = node1.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("key", "value", "trusting");
+
+    // Verify data synced
+    const mapOnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map.id);
+    expect(mapOnNode2.get("key")).toBe("value");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Map: map.core,
+        Group: group.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "server -> client | LOAD Map sessions: empty",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 5",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 1",
+        "server -> client | KNOWN Group sessions: header/5",
+        "server -> client | KNOWN Map sessions: header/1",
+      ]
+    `);
+  });
+
+  test("should handle disconnection correctly", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const peerId = "disconnect-test-peer";
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        id: peerId,
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      id: peerId,
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify peers are connected (same ID on both sides)
+    expect(node1.syncManager.peers["disconnect-test-peer"]).toBeDefined();
+    expect(node2.syncManager.peers["disconnect-test-peer"]).toBeDefined();
+
+    peer1.outgoing.close();
+
+    expect(node1.syncManager.peers["disconnect-test-peer"]).toBeUndefined();
+
+    await waitFor(() => {
+      expect(node2.syncManager.peers["disconnect-test-peer"]).toBeUndefined();
+    });
+  });
+
+  test("should ignore mismatched peer IDs in waitForConnection() when id filter is provided", async () => {
+    const { node } = setupTestNode();
+
+    const hostPeerId = "host-peer-id";
+    const wrongPeerId = "wrong-peer-id";
+
+    let acceptPromiseResolved = false;
+
+    // Mock worker that expects a different ID
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      // This should not resolve because the ID doesn't match
+      const acceptPromise = CojsonMessageChannel.acceptFromPort(port, {
+        id: wrongPeerId, // Expecting a different ID
+        role: "server",
+      });
+
+      // Set a timeout to detect if it's waiting
+      const timeoutPromise = new Promise<null>((resolve) =>
+        setTimeout(() => resolve(null), 100),
+      );
+
+      const result = await Promise.race([acceptPromise, timeoutPromise]);
+      if (result !== null) {
+        acceptPromiseResolved = true;
+        node.syncManager.addPeer(result);
+      }
+    });
+
+    // Expose with a different ID than what accept expects
+    CojsonMessageChannel.expose(mockWorker, {
+      id: hostPeerId,
+      role: "client",
+    });
+
+    // Wait a bit to ensure the accept didn't resolve
+    await new Promise((resolve) => setTimeout(resolve, 150));
+
+    // The accept should not have resolved because IDs don't match
+    expect(acceptPromiseResolved).toBe(false);
+  });
+
+  test("should sync data bidirectionally", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1
+    const group1 = node1.createGroup();
+    group1.addMember("everyone", "writer");
+    const map1 = group1.createMap();
+    map1.set("from", "node1", "trusting");
+
+    // Create data on node2
+    const group2 = node2.createGroup();
+    group2.addMember("everyone", "writer");
+    const map2 = group2.createMap();
+    map2.set("from", "node2", "trusting");
+
+    // Verify data synced in both directions
+    const map1OnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map1.id);
+    expect(map1OnNode2.get("from")).toBe("node1");
+
+    const map2OnNode1 = await loadCoValueOrFail<RawCoMap>(node1, map2.id);
+    expect(map2OnNode1.get("from")).toBe("node2");
+  });
+
+  test("should invoke onClose callback when connection closes", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let onCloseCalledOnHost = false;
+    let onCloseCalledOnWorker = false;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      const peer = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+        onClose: () => {
+          onCloseCalledOnWorker = true;
+        },
+      });
+      node2.syncManager.addPeer(peer);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+      onClose: () => {
+        onCloseCalledOnHost = true;
+      },
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Close the connection
+    peer1.outgoing.close();
+
+    // Wait for close to propagate
+    await waitFor(() => {
+      expect(onCloseCalledOnHost).toBe(true);
+    });
+
+    await waitFor(() => {
+      expect(onCloseCalledOnWorker).toBe(true);
+    });
+  });
+
+  test("should apply role configuration correctly", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify roles are correctly set
+    expect(peer1.role).toBe("client");
+    expect(peer2).not.toBeNull();
+    expect(peer2!.role).toBe("server");
+  });
+
+  test("should generate and use the same peer ID on both sides when not provided", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      node2.syncManager.addPeer(peer2);
+    });
+
+    // Don't provide an id - it should be auto-generated
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Verify peer1 has an auto-generated ID
+    expect(peer1.id).toMatch(/^channel_/);
+
+    // Verify both peers have the same ID
+    expect(peer2).not.toBeNull();
+    expect(peer2!.id).toBe(peer1.id);
+
+    // Verify the peer is accessible in both sync managers with the same ID
+    expect(node1.syncManager.peers[peer1.id]).toBeDefined();
+    expect(node2.syncManager.peers[peer1.id]).toBeDefined();
+  });
+
+  test("should handle delayed addPeer on accept side", async () => {
+    const { node: node1 } = setupTestNode();
+    const { node: node2 } = setupTestNode();
+
+    let peer2: Peer | null = null;
+
+    const delay = new Promise((resolve) => setTimeout(resolve, 50));
+
+    const mockWorker = createMockWorkerWithAccept(async (port) => {
+      peer2 = await CojsonMessageChannel.acceptFromPort(port, {
+        role: "server",
+      });
+      // Deliberately delay adding the peer
+      await delay;
+      node2.syncManager.addPeer(peer2);
+    });
+
+    const peer1 = await CojsonMessageChannel.expose(mockWorker, {
+      role: "client",
+    });
+    node1.syncManager.addPeer(peer1);
+
+    // Create data on node1 immediately (before node2 has added the peer)
+    const group = node1.createGroup();
+    group.addMember("everyone", "writer");
+    const map = group.createMap();
+    map.set("key", "value", "trusting");
+
+    await delay;
+
+    // Verify data synced despite the delay
+    const mapOnNode2 = await loadCoValueOrFail<RawCoMap>(node2, map.id);
+    expect(mapOnNode2.get("key")).toBe("value");
+  });
+});

package/src/tests/DeletedCoValuesEraserScheduler.test.ts

@@ -0,0 +1,185 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { logger } from "../logger.js";
+import { DeletedCoValuesEraserScheduler } from "../storage/DeletedCoValuesEraserScheduler.js";
+
+describe("DeletedCoValuesEraserScheduler", () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.clearAllTimers();
+    vi.useRealTimers();
+  });
+
+  test("scheduleStartupDrain runs once after startupDelayMs (when idle)", async () => {
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        return { hasMore: false };
+      },
+      opts: { throttleMs: 50, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.scheduleStartupDrain();
+
+    expect(runs).toBe(0);
+    await vi.advanceTimersByTimeAsync(10);
+    expect(runs).toBe(1);
+    scheduler.dispose();
+  });
+
+  test("onEnqueueDeletedCoValue is throttled (multiple enqueues -> one run)", async () => {
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        return { hasMore: false };
+      },
+      opts: { throttleMs: 30, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue();
+    scheduler.onEnqueueDeletedCoValue();
+    scheduler.onEnqueueDeletedCoValue();
+
+    expect(runs).toBe(0);
+    await vi.advanceTimersByTimeAsync(29);
+    expect(runs).toBe(0);
+    await vi.advanceTimersByTimeAsync(1);
+    expect(runs).toBe(1);
+
+    // Ensure no second run was scheduled by repeated enqueues in the same throttle window.
+    await vi.advanceTimersByTimeAsync(100);
+    expect(runs).toBe(1);
+
+    scheduler.dispose();
+  });
+
+  test("schedules follow-up phases while run reports hasMore=true", async () => {
+    let remaining = 3;
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        remaining -= 1;
+        return { hasMore: remaining > 0 };
+      },
+      opts: { throttleMs: 10, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue();
+
+    await vi.runAllTimersAsync();
+    expect(runs).toBe(3);
+    scheduler.dispose();
+  });
+
+  test("never runs run concurrently (re-entrancy guard via internal state machine)", async () => {
+    let concurrent = 0;
+    let maxConcurrent = 0;
+    let remaining = 2;
+
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        concurrent += 1;
+        maxConcurrent = Math.max(maxConcurrent, concurrent);
+
+        await new Promise<void>((resolve) => setTimeout(resolve, 30));
+        remaining -= 1;
+
+        concurrent -= 1;
+        return { hasMore: remaining > 0 };
+      },
+      opts: { throttleMs: 10, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue();
+    await vi.advanceTimersByTimeAsync(10); // start first run
+
+    // Even if we spam enqueues while active, they should be ignored.
+    scheduler.onEnqueueDeletedCoValue();
+    scheduler.onEnqueueDeletedCoValue();
+
+    await vi.runAllTimersAsync();
+    expect(remaining).toBe(0);
+    expect(maxConcurrent).toBe(1);
+
+    scheduler.dispose();
+  });
+
+  test("ignores enqueues while not idle, but schedules again once idle", async () => {
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        return { hasMore: false };
+      },
+      opts: { throttleMs: 30, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue(); // schedules first run
+    await vi.advanceTimersByTimeAsync(5);
+    scheduler.onEnqueueDeletedCoValue(); // should be ignored (not idle)
+
+    await vi.advanceTimersByTimeAsync(25);
+    expect(runs).toBe(1);
+
+    // Now idle again; next enqueue should schedule another run.
+    scheduler.onEnqueueDeletedCoValue();
+    await vi.advanceTimersByTimeAsync(30);
+    expect(runs).toBe(2);
+
+    scheduler.dispose();
+  });
+
+  test("dispose cancels any scheduled run", async () => {
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        return { hasMore: false };
+      },
+      opts: { throttleMs: 30, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue();
+    scheduler.dispose();
+
+    await vi.advanceTimersByTimeAsync(60);
+    expect(runs).toBe(0);
+  });
+
+  test("recovers when run throws (logs error and returns to idle so it can run again)", async () => {
+    const err = new Error("boom");
+    const errorSpy = vi.spyOn(logger, "error").mockImplementation(() => {});
+
+    let runs = 0;
+    const scheduler = new DeletedCoValuesEraserScheduler({
+      run: async () => {
+        runs += 1;
+        if (runs === 1) throw err;
+        return { hasMore: false };
+      },
+      opts: { throttleMs: 10, startupDelayMs: 10, followUpDelayMs: 10 },
+    });
+
+    scheduler.onEnqueueDeletedCoValue();
+    await vi.advanceTimersByTimeAsync(10);
+    expect(runs).toBe(1);
+
+    expect(errorSpy).toHaveBeenCalledWith(
+      "Error running deleted co values eraser scheduler",
+      expect.objectContaining({ err }),
+    );
+
+    // If the scheduler didn't reset back to idle after the error, this enqueue would be ignored.
+    scheduler.onEnqueueDeletedCoValue();
+    await vi.advanceTimersByTimeAsync(10);
+    expect(runs).toBe(2);
+
+    scheduler.dispose();
+    errorSpy.mockRestore();
+  });
+});