cojson 0.19.19 → 0.19.21

package/src/tests/SyncManager.processQueues.test.ts

@@ -0,0 +1,287 @@
+import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
+import {
+  SyncMessagesLog,
+  loadCoValueOrFail,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
+
+describe("SyncManager.processQueues", () => {
+  let jazzCloud: ReturnType<typeof setupTestNode>;
+
+  beforeEach(async () => {
+    SyncMessagesLog.clear();
+    jazzCloud = setupTestNode({
+      isSyncServer: true,
+    });
+  });
+
+  describe("incoming messages processing", () => {
+    test("should process incoming messages from peers", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+    });
+
+    test("should process multiple messages in sequence", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map1 = group.createMap();
+      const map2 = group.createMap();
+      const map3 = group.createMap();
+
+      map1.set("key", "value1", "trusting");
+      map2.set("key", "value2", "trusting");
+      map3.set("key", "value3", "trusting");
+
+      const [loadedMap1, loadedMap2, loadedMap3] = await Promise.all([
+        loadCoValueOrFail(client.node, map1.id),
+        loadCoValueOrFail(client.node, map2.id),
+        loadCoValueOrFail(client.node, map3.id),
+      ]);
+
+      expect(loadedMap1.get("key")).toEqual("value1");
+      expect(loadedMap2.get("key")).toEqual("value2");
+      expect(loadedMap3.get("key")).toEqual("value3");
+    });
+  });
+
+  describe("storage streaming processing", () => {
+    test("should process storage streaming callbacks", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+      const { storage } = client.addStorage();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // First load to populate storage
+      await loadCoValueOrFail(client.node, map.id);
+
+      // Restart and load from storage
+      client.restart();
+      client.connectToSyncServer();
+      client.addStorage({ storage });
+
+      SyncMessagesLog.clear();
+
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+
+      // Verify content came from storage
+      const storageMessages = SyncMessagesLog.messages.filter(
+        (msg) => msg.from === "storage" || msg.to === "storage",
+      );
+      expect(storageMessages.length).toBeGreaterThan(0);
+    });
+
+    test("should invoke streaming callbacks when pulled", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const callback = vi.fn();
+      storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.emit();
+
+      // Wait for processQueues to run
+      await waitFor(() => callback.mock.calls.length > 0);
+
+      expect(callback).toHaveBeenCalledTimes(1);
+    });
+
+    test("should process MEDIUM priority before LOW priority", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const order: string[] = [];
+      const lowCallback = () => order.push("low");
+      const mediumCallback = () => order.push("medium");
+      const highCallback = () => order.push("high");
+
+      // Push LOW first, then MEDIUM
+      storage.streamingQueue?.push(lowCallback, CO_VALUE_PRIORITY.LOW);
+      storage.streamingQueue?.push(mediumCallback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.push(highCallback, CO_VALUE_PRIORITY.HIGH);
+      storage.streamingQueue?.emit();
+
+      // Wait for both to be processed
+      await waitFor(() => order.length === 3);
+
+      // MEDIUM should be processed first
+      expect(order).toEqual(["high", "medium", "low"]);
+    });
+  });
+
+  describe("unified scheduling", () => {
+    test("should process both incoming messages and storage streaming", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+      const { storage } = client.addStorage();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // Queue a storage streaming callback
+      const streamingCallback = vi.fn();
+      storage.streamingQueue?.push(streamingCallback, CO_VALUE_PRIORITY.MEDIUM);
+      storage.streamingQueue?.emit();
+
+      // Load from server (incoming messages)
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+
+      expect(mapOnClient.get("hello")).toEqual("world");
+      expect(streamingCallback).toHaveBeenCalled();
+    });
+
+    test("should alternate between message queue and storage queue", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const order: string[] = [];
+
+      // Push multiple storage callbacks
+      storage.streamingQueue?.push(
+        () => order.push("storage1"),
+        CO_VALUE_PRIORITY.MEDIUM,
+      );
+      storage.streamingQueue?.push(
+        () => order.push("storage2"),
+        CO_VALUE_PRIORITY.MEDIUM,
+      );
+      storage.streamingQueue?.emit();
+
+      // Wait for processing
+      await waitFor(() => order.length === 2);
+
+      expect(order).toContain("storage1");
+      expect(order).toContain("storage2");
+    });
+  });
+
+  describe("processing flag", () => {
+    test("should prevent concurrent processQueues calls", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      let concurrentCalls = 0;
+      let maxConcurrentCalls = 0;
+
+      const callback = () => {
+        concurrentCalls++;
+        maxConcurrentCalls = Math.max(maxConcurrentCalls, concurrentCalls);
+        // Simulate some work
+        for (let i = 0; i < 1000; i++) {
+          Math.random();
+        }
+        concurrentCalls--;
+      };
+
+      // Push multiple callbacks
+      for (let i = 0; i < 10; i++) {
+        storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      }
+
+      // Emit multiple times to trigger multiple processQueues calls
+      storage.streamingQueue?.emit();
+      storage.streamingQueue?.emit();
+      storage.streamingQueue?.emit();
+
+      // Wait for all to complete
+      await waitFor(() => storage.streamingQueue?.isEmpty());
+
+      // Should never have more than 1 concurrent call
+      expect(maxConcurrentCalls).toBe(1);
+    });
+  });
+
+  describe("error handling", () => {
+    test("should continue processing after storage callback error", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const processed: string[] = [];
+
+      storage.streamingQueue?.push(() => {
+        processed.push("before");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.push(() => {
+        throw new Error("Test error");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.push(() => {
+        processed.push("after");
+      }, CO_VALUE_PRIORITY.MEDIUM);
+
+      storage.streamingQueue?.emit();
+
+      // Wait for processing to complete
+      await waitFor(() => storage.streamingQueue?.isEmpty());
+
+      // Both before and after should be processed despite error
+      expect(processed).toContain("before");
+      expect(processed).toContain("after");
+    });
+  });
+
+  describe("queue triggers", () => {
+    test("IncomingMessagesQueue.push should trigger processQueues", async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer();
+
+      const group = jazzCloud.node.createGroup();
+      const map = group.createMap();
+      map.set("hello", "world", "trusting");
+
+      // Loading should trigger message processing automatically
+      const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+      expect(mapOnClient.get("hello")).toEqual("world");
+    });
+
+    test("StorageStreamingQueue.emit should trigger processQueues", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage();
+
+      const callback = vi.fn();
+      storage.streamingQueue?.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+
+      // Before emit, callback should not be called
+      expect(callback).not.toHaveBeenCalled();
+
+      // After emit, processQueues should be triggered
+      storage.streamingQueue?.emit();
+
+      await waitFor(() => callback.mock.calls.length > 0);
+      expect(callback).toHaveBeenCalled();
+    });
+
+    test("setStorage should connect queue listener", async () => {
+      const client = setupTestNode();
+
+      // Before adding storage, there's no queue
+      const queueBefore = (
+        client.node.syncManager as any
+      ).getStorageStreamingQueue?.();
+      expect(queueBefore).toBeUndefined();
+
+      // After adding storage, queue should be available
+      const { storage } = client.addStorage();
+      const queueAfter = (
+        client.node.syncManager as any
+      ).getStorageStreamingQueue?.();
+      expect(queueAfter).toBe(storage.streamingQueue);
+    });
+  });
+});

package/src/tests/coValueCore.loadFromStorage.test.ts

@@ -36,6 +36,7 @@ function createMockStorage(
     ) => void;
     store?: (data: any, correctionCallback: any) => void;
     getKnownState?: (id: RawCoID) => any;
+    loadKnownState?: (id: string, callback: (knownState: any) => void) => void;
     waitForSync?: (id: string, coValue: any) => Promise<void>;
     trackCoValuesSyncState?: (
       operations: Array<{ id: RawCoID; peerId: PeerID; synced: boolean }>,
@@ -51,6 +52,8 @@ function createMockStorage(
     load: opts.load || vi.fn(),
     store: opts.store || vi.fn(),
     getKnownState: opts.getKnownState || vi.fn(),
+    loadKnownState:
+      opts.loadKnownState || vi.fn((id, callback) => callback(undefined)),
     waitForSync: opts.waitForSync || vi.fn().mockResolvedValue(undefined),
     trackCoValuesSyncState: opts.trackCoValuesSyncState || vi.fn(),
     getUnsyncedCoValueIDs: opts.getUnsyncedCoValueIDs || vi.fn(),

package/src/tests/knownState.lazyLoading.test.ts

@@ -0,0 +1,217 @@
+import { beforeEach, describe, expect, test, vi } from "vitest";
+import { RawCoID, SessionID } from "../ids";
+import { PeerID } from "../sync";
+import { StorageAPI } from "../storage/types";
+import { CoValueKnownState, peerHasAllContent } from "../knownState";
+import { createTestNode, createUnloadedCoValue } from "./testUtils";
+
+function createMockStorage(
+  opts: {
+    load?: (
+      id: RawCoID,
+      callback: (data: any) => void,
+      done: (found: boolean) => void,
+    ) => void;
+    store?: (data: any, correctionCallback: any) => void;
+    getKnownState?: (id: RawCoID) => any;
+    loadKnownState?: (id: string, callback: (knownState: any) => void) => void;
+    waitForSync?: (id: string, coValue: any) => Promise<void>;
+    trackCoValuesSyncState?: (
+      operations: Array<{ id: RawCoID; peerId: PeerID; synced: boolean }>,
+    ) => void;
+    getUnsyncedCoValueIDs?: (
+      callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+    ) => void;
+    stopTrackingSyncState?: (id: RawCoID) => void;
+    close?: () => Promise<unknown> | undefined;
+  } = {},
+): StorageAPI {
+  return {
+    load: opts.load || vi.fn(),
+    store: opts.store || vi.fn(),
+    getKnownState: opts.getKnownState || vi.fn(),
+    loadKnownState:
+      opts.loadKnownState || vi.fn((id, callback) => callback(undefined)),
+    waitForSync: opts.waitForSync || vi.fn().mockResolvedValue(undefined),
+    trackCoValuesSyncState: opts.trackCoValuesSyncState || vi.fn(),
+    getUnsyncedCoValueIDs: opts.getUnsyncedCoValueIDs || vi.fn(),
+    stopTrackingSyncState: opts.stopTrackingSyncState || vi.fn(),
+    close: opts.close || vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe("peerHasAllContent", () => {
+  const storageKnownState: CoValueKnownState = {
+    id: "co_test123" as RawCoID,
+    header: true,
+    sessions: {
+      ["session1" as SessionID]: 5,
+      ["session2" as SessionID]: 3,
+    },
+  };
+
+  test("returns false when peerKnownState is undefined", () => {
+    expect(peerHasAllContent(storageKnownState, undefined)).toBe(false);
+  });
+
+  test("returns false when peer does not have header but storage does", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: false,
+      sessions: {
+        ["session1" as SessionID]: 5,
+        ["session2" as SessionID]: 3,
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(false);
+  });
+
+  test("returns false when peer has fewer transactions in a session", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {
+        ["session1" as SessionID]: 3, // Less than storage's 5
+        ["session2" as SessionID]: 3,
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(false);
+  });
+
+  test("returns false when peer is missing a session", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {
+        ["session1" as SessionID]: 5,
+        // session2 is missing
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(false);
+  });
+
+  test("returns true when peer has exactly the same content", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {
+        ["session1" as SessionID]: 5,
+        ["session2" as SessionID]: 3,
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(true);
+  });
+
+  test("returns true when peer has more transactions than storage", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {
+        ["session1" as SessionID]: 10, // More than storage's 5
+        ["session2" as SessionID]: 5, // More than storage's 3
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(true);
+  });
+
+  test("returns true when peer has additional sessions", () => {
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {
+        ["session1" as SessionID]: 5,
+        ["session2" as SessionID]: 3,
+        ["session3" as SessionID]: 2, // Extra session not in storage
+      },
+    };
+    expect(peerHasAllContent(storageKnownState, peerKnownState)).toBe(true);
+  });
+
+  test("returns true when storage has empty sessions", () => {
+    const emptyStorageKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {},
+    };
+    const peerKnownState: CoValueKnownState = {
+      id: "co_test123" as RawCoID,
+      header: true,
+      sessions: {},
+    };
+    expect(peerHasAllContent(emptyStorageKnownState, peerKnownState)).toBe(
+      true,
+    );
+  });
+});
+
+describe("CoValueCore.getKnownStateFromStorage", () => {
+  function setup() {
+    const node = createTestNode();
+    const { coValue, id, header } = createUnloadedCoValue(node);
+    return { node, coValue, id, header };
+  }
+
+  test("returns undefined when storage is not configured", () => {
+    const { coValue } = setup();
+    const doneSpy = vi.fn();
+
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    expect(doneSpy).toHaveBeenCalledWith(undefined);
+  });
+
+  test("returns current knownState when CoValue is already available", () => {
+    const { node, coValue, header } = setup();
+    const storage = createMockStorage();
+    node.setStorage(storage);
+
+    // Make the CoValue available by providing header
+    coValue.provideHeader(header, undefined, false);
+
+    const doneSpy = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    expect(doneSpy).toHaveBeenCalledWith(
+      expect.objectContaining({
+        header: true,
+      }),
+    );
+  });
+
+  test("calls storage.loadKnownState when CoValue not in memory", () => {
+    const { node, coValue, id } = setup();
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback({
+        id,
+        header: true,
+        sessions: { session1: 5 },
+      });
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    const doneSpy = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    expect(loadKnownStateSpy).toHaveBeenCalledWith(id, expect.any(Function));
+    expect(doneSpy).toHaveBeenCalledWith({
+      id,
+      header: true,
+      sessions: { session1: 5 },
+    });
+  });
+
+  test("returns undefined when storage does not have the CoValue", () => {
+    const { node, coValue } = setup();
+    const loadKnownStateSpy = vi.fn((id, callback) => {
+      callback(undefined);
+    });
+    const storage = createMockStorage({ loadKnownState: loadKnownStateSpy });
+    node.setStorage(storage);
+
+    const doneSpy = vi.fn();
+    coValue.getKnownStateFromStorage(doneSpy);
+
+    expect(doneSpy).toHaveBeenCalledWith(undefined);
+  });
+});

package/src/tests/messagesTestUtils.ts

@@ -1,6 +1,7 @@
 import { CoValueCore, LocalNode } from "../exports";
 import { NewContentMessage, SyncMessage } from "../sync";
 import { CoValueKnownState } from "../knownState.js";
+import { LazyLoadMessage, LazyLoadResultMessage } from "./testUtils.js";
 
 function simplifySessions(msg: Pick<CoValueKnownState, "sessions" | "header">) {
   const count = Object.values(msg.sessions).reduce(
@@ -25,12 +26,14 @@ function simplifyNewContent(content: NewContentMessage["new"]) {
     .join(" | ");
 }
 
+type TestMessage = SyncMessage | LazyLoadMessage | LazyLoadResultMessage;
+
 export function toSimplifiedMessages(
   coValues: Record<string, CoValueCore>,
   messages: {
     from: string;
     to: string;
-    msg: SyncMessage;
+    msg: TestMessage;
   }[],
 ) {
   function getCoValue(id: string) {
@@ -43,7 +46,7 @@ export function toSimplifiedMessages(
     return `unknown/${id}`;
   }
 
-  function toDebugString(from: string, to: string, msg: SyncMessage) {
+  function toDebugString(from: string, to: string, msg: TestMessage) {
     switch (msg.action) {
       case "known":
         return `${from} -> ${to} | KNOWN ${msg.isCorrection ? "CORRECTION " : ""}${getCoValue(msg.id)} sessions: ${simplifySessions(msg)}`;
@@ -53,6 +56,10 @@ export function toSimplifiedMessages(
         return `${from} -> ${to} | DONE ${getCoValue(msg.id)}`;
       case "content":
         return `${from} -> ${to} | CONTENT ${getCoValue(msg.id)} header: ${Boolean(msg.header)} new: ${simplifyNewContent(msg.new)}${msg.expectContentUntil ? ` expectContentUntil: ${simplifySessions({ sessions: msg.expectContentUntil, header: true })}` : ""}`;
+      case "lazyLoad":
+        return `${from} -> ${to} | GET_KNOWN_STATE ${getCoValue(msg.id)}`;
+      case "lazyLoadResult":
+        return `${from} -> ${to} | GET_KNOWN_STATE_RESULT ${getCoValue(msg.id)} sessions: ${simplifySessions(msg)}`;
     }
   }
 
@@ -75,7 +82,7 @@ export function debugMessages(
   messages: {
     from: string;
     to: string;
-    msg: SyncMessage;
+    msg: TestMessage;
   }[],
 ) {
   console.log(toSimplifiedMessages(coValues, messages));

package/src/tests/setup.ts

@@ -0,0 +1,4 @@
+import { cojsonInternals } from "../exports.js";
+
+// Use a very high budget to avoid that slow tests fail due to the budget being exceeded.
+cojsonInternals.setIncomingMessagesTimeBudget(10000); // 10 seconds

package/src/tests/sync.garbageCollection.test.ts

@@ -1,13 +1,11 @@
-import { assert, beforeEach, describe, expect, test, vi } from "vitest";
+import { beforeEach, describe, expect, test } from "vitest";
 
 import { setGarbageCollectorMaxAge } from "../config";
-import { emptyKnownState } from "../exports";
 import {
   SyncMessagesLog,
   TEST_NODE_CONFIG,
   loadCoValueOrFail,
   setupTestNode,
-  waitFor,
 } from "./testUtils";
 
 // We want to simulate a real world communication that happens asynchronously