cojson 0.19.18 → 0.19.20

package/src/tests/StorageStreamingQueue.test.ts

@@ -0,0 +1,276 @@
+import { describe, expect, test, vi } from "vitest";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
+
+describe("StorageStreamingQueue", () => {
+  describe("constructor", () => {
+    test("should initialize with empty queues", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.isEmpty()).toBe(true);
+    });
+  });
+
+  describe("push", () => {
+    test("should add MEDIUM priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should add LOW priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should add HIGH priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.HIGH);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should accept multiple entries", () => {
+      const queue = new StorageStreamingQueue();
+      const entry1 = () => {};
+      const entry2 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry2, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.pull()).toBe(entry1);
+      expect(queue.pull()).toBe(entry2);
+    });
+  });
+
+  describe("pull", () => {
+    test("should return undefined for empty queue", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.pull()).toBeUndefined();
+    });
+
+    test("should return and remove entry from queue", () => {
+      const queue = new StorageStreamingQueue();
+      const entry = () => {};
+
+      queue.push(entry, CO_VALUE_PRIORITY.MEDIUM);
+      const pulled = queue.pull();
+
+      expect(pulled).toBe(entry);
+      expect(queue.isEmpty()).toBe(true);
+    });
+
+    test("should pull HIGH priority before MEDIUM and LOW", () => {
+      const queue = new StorageStreamingQueue();
+      const lowEntry = () => {};
+      const mediumEntry = () => {};
+      const highEntry = () => {};
+
+      // Push in reverse order
+      queue.push(lowEntry, CO_VALUE_PRIORITY.LOW);
+      queue.push(mediumEntry, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(highEntry, CO_VALUE_PRIORITY.HIGH);
+
+      // Should pull HIGH first, then MEDIUM, then LOW
+      expect(queue.pull()).toBe(highEntry);
+      expect(queue.pull()).toBe(mediumEntry);
+      expect(queue.pull()).toBe(lowEntry);
+    });
+
+    test("should pull MEDIUM priority before LOW priority", () => {
+      const queue = new StorageStreamingQueue();
+      const lowEntry = () => {};
+      const mediumEntry = () => {};
+
+      // Push LOW first, then MEDIUM
+      queue.push(lowEntry, CO_VALUE_PRIORITY.LOW);
+      queue.push(mediumEntry, CO_VALUE_PRIORITY.MEDIUM);
+
+      // Should pull MEDIUM first
+      expect(queue.pull()).toBe(mediumEntry);
+      expect(queue.pull()).toBe(lowEntry);
+    });
+
+    test("should pull entries in FIFO order within same priority", () => {
+      const queue = new StorageStreamingQueue();
+      const entry1 = () => {};
+      const entry2 = () => {};
+      const entry3 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry2, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry3, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.pull()).toBe(entry1);
+      expect(queue.pull()).toBe(entry2);
+      expect(queue.pull()).toBe(entry3);
+    });
+
+    test("should handle interleaved priorities correctly", () => {
+      const queue = new StorageStreamingQueue();
+      const low1 = () => {};
+      const medium1 = () => {};
+      const high1 = () => {};
+      const low2 = () => {};
+      const medium2 = () => {};
+      const high2 = () => {};
+
+      queue.push(low1, CO_VALUE_PRIORITY.LOW);
+      queue.push(medium1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(high1, CO_VALUE_PRIORITY.HIGH);
+      queue.push(low2, CO_VALUE_PRIORITY.LOW);
+      queue.push(medium2, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(high2, CO_VALUE_PRIORITY.HIGH);
+
+      // All HIGH should come first, in order
+      expect(queue.pull()).toBe(high1);
+      expect(queue.pull()).toBe(high2);
+      // Then all MEDIUM, in order
+      expect(queue.pull()).toBe(medium1);
+      expect(queue.pull()).toBe(medium2);
+      // Then all LOW, in order
+      expect(queue.pull()).toBe(low1);
+      expect(queue.pull()).toBe(low2);
+    });
+  });
+
+  describe("isEmpty", () => {
+    test("should return true for empty queue", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.isEmpty()).toBe(true);
+    });
+
+    test("should return false when MEDIUM queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return false when LOW queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return false when HIGH queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.HIGH);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return true after all entries are pulled", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+
+      queue.pull();
+      queue.pull();
+
+      expect(queue.isEmpty()).toBe(true);
+    });
+  });
+
+  describe("callback invocation", () => {
+    test("should not invoke callback when pushed", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    test("should not invoke callback when pulled", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      queue.pull();
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    test("should allow caller to invoke callback after pull", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      const pulled = queue.pull();
+
+      expect(callback).not.toHaveBeenCalled();
+
+      pulled?.();
+
+      expect(callback).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("setListener and emit", () => {
+    test("should call listener when emit is called", () => {
+      const queue = new StorageStreamingQueue();
+      const listener = vi.fn();
+
+      queue.setListener(listener);
+      queue.emit();
+
+      expect(listener).toHaveBeenCalledTimes(1);
+    });
+
+    test("should not throw when emit is called without listener", () => {
+      const queue = new StorageStreamingQueue();
+
+      expect(() => queue.emit()).not.toThrow();
+    });
+
+    test("should call listener multiple times on multiple emits", () => {
+      const queue = new StorageStreamingQueue();
+      const listener = vi.fn();
+
+      queue.setListener(listener);
+      queue.emit();
+      queue.emit();
+      queue.emit();
+
+      expect(listener).toHaveBeenCalledTimes(3);
+    });
+
+    test("should use latest listener when setListener is called multiple times", () => {
+      const queue = new StorageStreamingQueue();
+      const listener1 = vi.fn();
+      const listener2 = vi.fn();
+
+      queue.setListener(listener1);
+      queue.setListener(listener2);
+      queue.emit();
+
+      expect(listener1).not.toHaveBeenCalled();
+      expect(listener2).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("edge cases", () => {
+    test("should handle alternating push and pull operations", () => {
+      const queue = new StorageStreamingQueue();
+
+      const entry1 = () => {};
+      const entry2 = () => {};
+      const entry3 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.pull()).toBe(entry1);
+
+      queue.push(entry2, CO_VALUE_PRIORITY.LOW);
+      expect(queue.pull()).toBe(entry2);
+
+      expect(queue.isEmpty()).toBe(true);
+
+      queue.push(entry3, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.pull()).toBe(entry3);
+    });
+  });
+});

package/src/tests/SyncManager.processQueues.test.ts

@@ -0,0 +1,287 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
+import {
+  SyncMessagesLog,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
+
+describe("SyncManager.processQueues", () => {
+  let jazzCloud: ReturnType<typeof setupTestNode>;
+
+  beforeEach(async () => {
+    SyncMessagesLog.clear();
+    jazzCloud = setupTestNode({
+      isSyncServer: true,
+    });
+  });
+
+  describe("incoming messages processing", () => {
+    test("should process incoming messages from peers", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+    });
+
+    test("should process multiple messages in sequence", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map1 = group.createMap();
+      const map2 = group.createMap();
+      const map3 = group.createMap();
+
+      map1.set("key", "value1", "trusting");
+      map2.set("key", "value2", "trusting");
+      map3.set("key", "value3", "trusting");
+
+      const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
+        loadCoValueOrFail(client.node, map1.id),
+        loadCoValueOrFail(client.node, map2.id),
+        loadCoValueOrFail(client.node, map3.id),
+      ]);
+
+      expect(loadedMap1.get("key")).toEqual("value1");
+      expect(loadedMap2.get("key")).toEqual("value2");
+      expect(loadedMap3.get("key")).toEqual("value3");
+    });
+  });
+
+  describe("storage streaming processing", () => {
+    test("should process storage streaming callbacks", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+      const { storage } = client.addStorage();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // First load to populate storage
+      await loadCoValueOrFail(client.node, map.id);
+
+      // Restart and load from storage
+      client.restart();
+      client.connectToSyncServer();
+      client.addStorage({ storage });
+
+      SyncMessagesLog.clear();
+
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+
+      // Verify content came from storage
+      const storageMessages = SyncMessagesLog.messages.filter(
+        (msg) => msg.from === "storage" || msg.to === "storage",
+      );
+      expect(storageMessages.length).toBeGreaterThan(0);
+    });
+
+    test("should invoke streaming callbacks when pulled", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const callback = vi.fn();
+      storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.emit();
+
+      // Wait for processQueues to run
+      await waitFor(() => callback.mock.calls.length > 0);
+
+      expect(callback).toHaveBeenCalledTimes(1);
+    });
+
+    test("should process MEDIUM priority before LOW priority", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const order: string[] = [];
+      const lowCallback = () => order.push("low");
+      const mediumCallback = () => order.push("medium");
+      const highCallback = () => order.push("high");
+
+      // Push LOW first, then MEDIUM
+      storage.streamingQueue?.push(lowCallback, CO_VALUE_PRIORITY.LOW);
+      storage.streamingQueue?.push(mediumCallback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.push(highCallback, CO_VALUE_PRIORITY.HIGH);
+      storage.streamingQueue?.emit();
+
+      // Wait for both to be processed
+      await waitFor(() => order.length === 3);
+
+      // MEDIUM should be processed first
+      expect(order).toEqual(["high", "medium", "low"]);
+    });
+  });
+
+  describe("unified scheduling", () => {
+    test("should process both incoming messages and storage streaming", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+      const { storage } = client.addStorage();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // Queue a storage streaming callback
+      const streamingCallback = vi.fn();
+      storage.streamingQueue?.push(streamingCallback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.emit();
+
+      // Load from server (incoming messages)
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+
+      expect(mapOnClient.get("hello")).toEqual("world");
+      expect(streamingCallback).toHaveBeenCalled();
+    });
+
+    test("should alternate between message queue and storage queue", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const order: string[] = [];
+
+      // Push multiple storage callbacks
+      storage.streamingQueue?.push(
+        () => order.push("storage1"),
+        CO_VALUE_PRIORITY.MEDIUM,
+      );
+      storage.streamingQueue?.push(
+        () => order.push("storage2"),
+        CO_VALUE_PRIORITY.MEDIUM,
+      );
+      storage.streamingQueue?.emit();
+
+      // Wait for processing
+      await waitFor(() => order.length === 2);
+
+      expect(order).toContain("storage1");
+      expect(order).toContain("storage2");
+    });
+  });
+
+  describe("processing flag", () => {
+    test("should prevent concurrent processQueues calls", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      let concurrentCalls = 0;
+      let maxConcurrentCalls = 0;
+
+      const callback = () => {
+        concurrentCalls++;
+        maxConcurrentCalls = Math.max(maxConcurrentCalls, concurrentCalls);
+        // Simulate some work
+        for (let i = 0; i < 1000; i++) {
+          Math.random();
+        }
+        concurrentCalls--;
+      };
+
+      // Push multiple callbacks
+      for (let i = 0; i < 10; i++) {
+        storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      }
+
+      // Emit multiple times to trigger multiple processQueues calls
+      storage.streamingQueue?.emit();
+      storage.streamingQueue?.emit();
+      storage.streamingQueue?.emit();
+
+      // Wait for all to complete
+      await waitFor(() => storage.streamingQueue?.isEmpty());
+
+      // Should never have more than 1 concurrent call
+      expect(maxConcurrentCalls).toBe(1);
+    });
+  });
+
+  describe("error handling", () => {
+    test("should continue processing after storage callback error", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const processed: string[] = [];
+
+      storage.streamingQueue?.push(() => {
+        processed.push("before");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.push(() => {
+        throw new Error("Test error");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.push(() => {
+        processed.push("after");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.emit();
+
+      // Wait for processing to complete
+      await waitFor(() => storage.streamingQueue?.isEmpty());
+
+      // Both before and after should be processed despite error
+      expect(processed).toContain("before");
+      expect(processed).toContain("after");
+    });
+  });
+
+  describe("queue triggers", () => {
+    test("IncomingMessagesQueue.push should trigger processQueues", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // Loading should trigger message processing automatically
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+    });
+
+    test("StorageStreamingQueue.emit should trigger processQueues", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const callback = vi.fn();
+      storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+
+      // Before emit, callback should not be called
+      expect(callback).not.toHaveBeenCalled();
+
+      // After emit, processQueues should be triggered
+      storage.streamingQueue?.emit();
+
+      await waitFor(() => callback.mock.calls.length > 0);
+      expect(callback).toHaveBeenCalled();
+    });
+
+    test("setStorage should connect queue listener", async () => {
+      const client = setupTestNode();
+
+      // Before adding storage, there's no queue
+      const queueBefore = (
+        client.node.syncManager as any
+      ).getStorageStreamingQueue?.();
+      expect(queueBefore).toBeUndefined();
+
+      // After adding storage, queue should be available
+      const { storage } = client.addStorage();
+      const queueAfter = (
+        client.node.syncManager as any
+      ).getStorageStreamingQueue?.();
+      expect(queueAfter).toBe(storage.streamingQueue);
+    });
+  });
+});

package/src/tests/SyncStateManager.test.ts

@@ -78,10 +78,12 @@ describe("SyncStateManager", () => {
     const updateToStorageSpy: PeerSyncStateListenerCallback = vi.fn();
     const unsubscribe1 = subscriptionManager.subscribeToPeerUpdates(
       peerState.id,
+      map.core.id,
       updateToJazzCloudSpy,
     );
     const unsubscribe2 = subscriptionManager.subscribeToPeerUpdates(
       serverPeer.id,
+      group.core.id,
       updateToStorageSpy,
     );
 
@@ -141,6 +143,7 @@ describe("SyncStateManager", () => {
     const unsubscribe1 = subscriptionManager.subscribeToUpdates(anyUpdateSpy);
     const unsubscribe2 = subscriptionManager.subscribeToPeerUpdates(
       peerState.id,
+      map.core.id,
       anyUpdateSpy,
     );
 

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -1,5 +1,6 @@
 import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
 import { RawCoID } from "../ids";
+import { PeerID } from "../sync";
 import { StorageAPI } from "../storage/types";
 import {
   createTestMetricReader,
@@ -36,6 +37,13 @@ function createMockStorage(
     store?: (data: any, correctionCallback: any) => void;
     getKnownState?: (id: RawCoID) => any;
     waitForSync?: (id: string, coValue: any) => Promise<void>;
+    trackCoValuesSyncState?: (
+      operations: Array<{ id: RawCoID; peerId: PeerID; synced: boolean }>,
+    ) => void;
+    getUnsyncedCoValueIDs?: (
+      callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+    ) => void;
+    stopTrackingSyncState?: (id: RawCoID) => void;
     close?: () => Promise<unknown> | undefined;
   } = {},
 ): StorageAPI {
@@ -44,6 +52,9 @@ function createMockStorage(
     store: opts.store || vi.fn(),
     getKnownState: opts.getKnownState || vi.fn(),
     waitForSync: opts.waitForSync || vi.fn().mockResolvedValue(undefined),
+    trackCoValuesSyncState: opts.trackCoValuesSyncState || vi.fn(),
+    getUnsyncedCoValueIDs: opts.getUnsyncedCoValueIDs || vi.fn(),
+    stopTrackingSyncState: opts.stopTrackingSyncState || vi.fn(),
     close: opts.close || vi.fn().mockResolvedValue(undefined),
   };
 }

package/src/tests/setup.ts

@@ -0,0 +1,4 @@
+import { cojsonInternals } from "../exports.js";
+
+// Use a very high budget to avoid that slow tests fail due to the budget being exceeded.
+cojsonInternals.setIncomingMessagesTimeBudget(10000); // 10 seconds

package/src/tests/sync.garbageCollection.test.ts

@@ -1,13 +1,11 @@
-import { assert, beforeEach, describe, expect, test, vi } from "vitest";
+import { beforeEach, describe, expect, test } from "vitest";
 
 import { setGarbageCollectorMaxAge } from "../config";
-import { emptyKnownState } from "../exports";
 import {
   SyncMessagesLog,
   TEST_NODE_CONFIG,
   loadCoValueOrFail,
   setupTestNode,
-  waitFor,
 } from "./testUtils";
 
 // We want to simulate a real world communication that happens asynchronously