cojson 0.20.9 → 0.20.10

package/src/tests/OutgoingLoadQueue.test.ts

@@ -1,4 +1,12 @@
-import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import {
+  afterEach,
+  beforeEach,
+  describe,
+  expect,
+  onTestFinished,
+  test,
+  vi,
+} from "vitest";
 import {
   CO_VALUE_LOADING_CONFIG,
   setMaxInFlightLoadsPerPeer,
@@ -195,6 +203,26 @@ describe("OutgoingLoadQueue", () => {
       expect(duplicateCallbackCount).toBe(0);
     });
 
+    test("should skip duplicate enqueue for same CoValue ID while in-flight", () => {
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const otherNode = createTestNode();
+      const group = node.createGroup();
+
+      const map = group.createMap();
+      const sameIdCoValue = otherNode.getCoValue(map.id);
+
+      let duplicateCallbackCount = 0;
+
+      queue.enqueue(map.core, () => {});
+      queue.enqueue(sameIdCoValue, () => {
+        duplicateCallbackCount += 1;
+      });
+
+      expect(queue.inFlightCount).toBe(1);
+      expect(duplicateCallbackCount).toBe(0);
+    });
+
     test("should skip duplicate enqueue for same CoValue ID", () => {
       setMaxInFlightLoadsPerPeer(1);
       const queue = new OutgoingLoadQueue(TEST_PEER_ID);
@@ -417,6 +445,34 @@ describe("OutgoingLoadQueue", () => {
       expect(queue.inFlightCount).toBe(1);
     });
 
+    test("should complete in-flight request by CoValue ID even with different instance", () => {
+      setMaxInFlightLoadsPerPeer(1);
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const otherNode = createTestNode();
+      const group = node.createGroup();
+
+      const map = group.createMap();
+      const nextMap = group.createMap();
+      const sameIdCoValue = otherNode.getCoValue(map.id);
+
+      let nextCallbackCalled = false;
+
+      queue.enqueue(map.core, () => {});
+      queue.enqueue(nextMap.core, () => {
+        nextCallbackCalled = true;
+      });
+
+      expect(queue.inFlightCount).toBe(1);
+      expect(nextCallbackCalled).toBe(false);
+
+      // Complete using a different CoValue instance with the same ID.
+      queue.trackComplete(sameIdCoValue);
+
+      expect(nextCallbackCalled).toBe(true);
+      expect(queue.inFlightCount).toBe(1);
+    });
+
     test("should allow re-enqueue after completion", () => {
       setMaxInFlightLoadsPerPeer(1);
       const queue = new OutgoingLoadQueue(TEST_PEER_ID);
@@ -445,7 +501,7 @@ describe("OutgoingLoadQueue", () => {
       expect(secondCallbackCount).toBe(1);
     });
 
-    test("should not complete if CoValue is streaming", () => {
+    test("should not complete if CoValue is streaming for content updates", () => {
       setMaxInFlightLoadsPerPeer(1);
       const queue = new OutgoingLoadQueue(TEST_PEER_ID);
       const node = createTestNode();
@@ -468,7 +524,7 @@ describe("OutgoingLoadQueue", () => {
       expect(callback2Called).toBe(false);
 
       // trackComplete should not complete because map1 is streaming
-      queue.trackComplete(map1.core);
+      queue.trackComplete(map1.core, "content");
 
       // Should still be in-flight and pending should not be processed
       expect(queue.inFlightCount).toBe(1);
@@ -479,7 +535,35 @@ describe("OutgoingLoadQueue", () => {
       vi.spyOn(map1.core, "isStreaming").mockReturnValue(false);
 
       // Now trackComplete should work
-      queue.trackComplete(map1.core);
+      queue.trackComplete(map1.core, "content");
+
+      expect(queue.inFlightCount).toBe(1); // map2 is now in-flight
+      expect(callback2Called).toBe(true);
+      expect(queue.pendingCount).toBe(0);
+    });
+
+    test("should complete if CoValue is streaming for known-state updates", () => {
+      setMaxInFlightLoadsPerPeer(1);
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const group = node.createGroup();
+
+      const map1 = group.createMap();
+      const map2 = group.createMap();
+
+      vi.spyOn(map1.core, "isStreaming").mockReturnValue(true);
+
+      let callback2Called = false;
+
+      queue.enqueue(map1.core, () => {});
+      queue.enqueue(map2.core, () => {
+        callback2Called = true;
+      });
+
+      expect(queue.inFlightCount).toBe(1);
+      expect(callback2Called).toBe(false);
+
+      queue.trackComplete(map1.core, "known");
 
       expect(queue.inFlightCount).toBe(1); // map2 is now in-flight
       expect(callback2Called).toBe(true);
@@ -731,62 +815,145 @@ describe("OutgoingLoadQueue", () => {
 
     test("should balance push/pull metrics when clearing pending items", async () => {
       const metricReader = createTestMetricReader();
+      onTestFinished(tearDownTestMetricReader);
 
-      try {
-        setMaxInFlightLoadsPerPeer(1);
-        const queue = new OutgoingLoadQueue(TEST_PEER_ID);
-        const node = createTestNode();
-        const group = node.createGroup();
-
-        // Block the queue first (this item is pushed then immediately shifted to go in-flight)
-        const blockerMap = group.createMap();
-        queue.enqueue(blockerMap.core, () => {});
-
-        // Enqueue multiple items that will be pending
-        const map1 = group.createMap();
-        const map2 = group.createMap();
-        const map3 = group.createMap();
-
-        queue.enqueue(map1.core, () => {});
-        queue.enqueue(map2.core, () => {});
-        queue.enqueue(map3.core, () => {});
-
-        expect(queue.pendingCount).toBe(3);
-
-        // Get metrics before clear
-        // 4 items pushed (blocker + 3 pending), 1 pulled (blocker was shifted to go in-flight)
-        const pushedBefore = await metricReader.getMetricValue(
-          "jazz.messagequeue.load-requests-queue.pushed",
-          { priority: "high" },
-        );
-        const pulledBefore = await metricReader.getMetricValue(
-          "jazz.messagequeue.load-requests-queue.pulled",
-          { priority: "high" },
-        );
-
-        expect(pushedBefore).toBe(4);
-        expect(pulledBefore).toBe(1);
-
-        // Clear the queue - this should drain pending items, incrementing pulled counter
-        queue.clear();
-
-        // Get metrics after clear - pushed and pulled should now be equal
-        const pushedAfter = await metricReader.getMetricValue(
-          "jazz.messagequeue.load-requests-queue.pushed",
-          { priority: "high" },
-        );
-        const pulledAfter = await metricReader.getMetricValue(
-          "jazz.messagequeue.load-requests-queue.pulled",
-          { priority: "high" },
-        );
-
-        // All 4 pushed items should now be pulled (balanced)
-        expect(pushedAfter).toBe(4);
-        expect(pulledAfter).toBe(4);
-        expect(pushedAfter).toBe(pulledAfter);
-      } finally {
-        tearDownTestMetricReader();
-      }
+      setMaxInFlightLoadsPerPeer(1);
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const group = node.createGroup();
+
+      // Block the queue first (this item is pushed then immediately shifted to go in-flight)
+      const blockerMap = group.createMap();
+      queue.enqueue(blockerMap.core, () => {});
+
+      // Enqueue multiple items that will be pending
+      const map1 = group.createMap();
+      const map2 = group.createMap();
+      const map3 = group.createMap();
+
+      queue.enqueue(map1.core, () => {});
+      queue.enqueue(map2.core, () => {});
+      queue.enqueue(map3.core, () => {});
+
+      expect(queue.pendingCount).toBe(3);
+
+      // Get metrics before clear
+      // 4 items pushed (blocker + 3 pending), 1 pulled (blocker was shifted to go in-flight)
+      const pushedBefore = await metricReader.getMetricValue(
+        "jazz.messagequeue.load-requests-queue.pushed",
+        { priority: "high" },
+      );
+      const pulledBefore = await metricReader.getMetricValue(
+        "jazz.messagequeue.load-requests-queue.pulled",
+        { priority: "high" },
+      );
+
+      expect(pushedBefore).toBe(4);
+      expect(pulledBefore).toBe(1);
+
+      // Clear the queue - this should drain pending items, incrementing pulled counter
+      queue.clear();
+
+      // Get metrics after clear - pushed and pulled should now be equal
+      const pushedAfter = await metricReader.getMetricValue(
+        "jazz.messagequeue.load-requests-queue.pushed",
+        { priority: "high" },
+      );
+      const pulledAfter = await metricReader.getMetricValue(
+        "jazz.messagequeue.load-requests-queue.pulled",
+        { priority: "high" },
+      );
+
+      // All 4 pushed items should now be pulled (balanced)
+      expect(pushedAfter).toBe(4);
+      expect(pulledAfter).toBe(4);
+      expect(pushedAfter).toBe(pulledAfter);
+    });
+
+    test("should track in-flight loads with OpenTelemetry", async () => {
+      const metricReader = createTestMetricReader();
+      onTestFinished(tearDownTestMetricReader);
+
+      setMaxInFlightLoadsPerPeer(2);
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const group = node.createGroup();
+
+      const map1 = group.createMap();
+      const map2 = group.createMap();
+
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(0);
+
+      queue.enqueue(map1.core, () => {});
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(1);
+
+      queue.enqueue(map2.core, () => {});
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(2);
+
+      queue.trackComplete(map1.core);
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(1);
+
+      queue.clear();
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(0);
+    });
+
+    test("should decrement in-flight metric when a load times out", async () => {
+      CO_VALUE_LOADING_CONFIG.TIMEOUT = 5;
+
+      const metricReader = createTestMetricReader();
+      onTestFinished(tearDownTestMetricReader);
+
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const coValue = node.getCoValue("co_zTestMetricTimeout00001" as any);
+
+      queue.enqueue(coValue, () => {});
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(1);
+
+      await new Promise((resolve) => setTimeout(resolve, 20));
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(0);
+    });
+
+    test("should not change in-flight metric when completing unknown CoValue", async () => {
+      const metricReader = createTestMetricReader();
+      onTestFinished(tearDownTestMetricReader);
+
+      const queue = new OutgoingLoadQueue(TEST_PEER_ID);
+      const node = createTestNode();
+      const group = node.createGroup();
+      const knownMap = group.createMap();
+      const unknownCoValue = node.getCoValue(
+        "co_zTestUnknownMetric0001" as any,
+      );
+
+      queue.enqueue(knownMap.core, () => {});
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(1);
+
+      queue.trackComplete(unknownCoValue);
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(1);
+
+      queue.trackComplete(knownMap.core);
+      expect(
+        await metricReader.getMetricValue("jazz.loadqueue.outgoing.inflight"),
+      ).toBe(0);
     });
   });
 

package/src/tests/SQLiteClientAsync.test.ts

@@ -36,7 +36,7 @@ describe("SQLiteClientAsync", () => {
 
       const signatures = await dbClient.getSignatures(0, 0);
       expect(signatures.length).toBe(10);
-      signatures.forEach(async ({ signature }, i) => {
+      signatures.forEach(({ signature }, i) => {
         expect(signature).toBe(`signature_z${i}`);
       });
     });

package/src/tests/StorageApiAsync.test.ts

@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, test, vi } from "vitest";
-import { CoID, RawCoMap, logger } from "../exports.js";
+import { CoID, RawCoID, RawCoMap, logger } from "../exports.js";
 import { CoValueCore } from "../exports.js";
 import { NewContentMessage } from "../sync.js";
 import {
@@ -1413,4 +1413,164 @@ describe("StorageApiAsync", () => {
       expect(dbClientSpy).toHaveBeenCalledTimes(0);
     });
   });
+
+  describe("getCoValueIDs", () => {
+    test("should return empty array when storage has no CoValues", async () => {
+      const client = setupTestNode();
+      const { storage } = await client.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+      });
+
+      const ids = await new Promise<{ id: RawCoID }[]>((resolve) => {
+        storage.getCoValueIDs(100, 0, resolve);
+      });
+
+      expect(ids).toEqual([]);
+    });
+
+    test("should return CoValue IDs in batch after storing CoValues", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      await fixtures.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      const client = setupTestNode();
+      const { storage } = await client.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      // Create CoValues and sync to storage
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap();
+      map.set("key", "value", "trusting");
+      await map.core.waitForSync();
+
+      const ids = await new Promise<{ id: RawCoID }[]>((resolve) => {
+        storage.getCoValueIDs(100, 0, resolve);
+      });
+
+      expect(ids.map((e) => e.id)).toContain(group.id);
+      expect(ids.map((e) => e.id)).toContain(map.id);
+      expect(ids.length).toEqual(2);
+    });
+
+    test("should paginate when there are more CoValues than the limit and return each ID only once", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      await fixtures.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      const client = setupTestNode();
+      const { storage } = await client.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      // Create more CoValues than the page size (1 group + 4 maps = 5 CoValues, limit = 2)
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      const expectedIds = new Set<RawCoID>([group.id]);
+      const maps: ReturnType<typeof group.createMap>[] = [];
+      for (let i = 0; i < 4; i++) {
+        const map = group.createMap();
+        map.set(`key${i}`, `value${i}`, "trusting");
+        maps.push(map);
+        expectedIds.add(map.id);
+      }
+      await maps[maps.length - 1]!.core.waitForSync();
+
+      const limit = 2;
+      const allIds: RawCoID[] = [];
+      await new Promise<void>((resolve) => {
+        const fetchBatch = (offset: number) => {
+          storage.getCoValueIDs(limit, offset, (batch) => {
+            for (const { id } of batch) {
+              allIds.push(id);
+            }
+            if (batch.length >= limit) {
+              fetchBatch(offset + batch.length);
+            } else {
+              resolve();
+            }
+          });
+        };
+        fetchBatch(0);
+      });
+
+      expect(allIds).toHaveLength(expectedIds.size);
+      const seen = new Set<RawCoID>();
+      for (const id of allIds) {
+        expect(seen.has(id)).toBe(false);
+        seen.add(id);
+        expect(expectedIds.has(id)).toBe(true);
+      }
+    });
+  });
+
+  describe("getCoValueCount", () => {
+    test("should return 0 when storage has no CoValues", async () => {
+      const client = setupTestNode();
+      const { storage } = await client.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+      });
+
+      const count = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+
+      expect(count).toBe(0);
+    });
+
+    test("should return CoValue count after storing CoValues", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      await fixtures.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      const client = setupTestNode();
+      const { storage } = await client.addAsyncStorage({
+        ourName: "test",
+        storageName: "test-storage",
+        filename: dbPath,
+      });
+
+      const countEmpty = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countEmpty).toBe(0);
+
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      const countOne = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countOne).toBe(1);
+
+      const map = group.createMap();
+      map.set("key", "value", "trusting");
+      await map.core.waitForSync();
+
+      const countTwo = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countTwo).toBe(2);
+    });
+  });
 });

package/src/tests/StorageApiSync.test.ts

@@ -1237,4 +1237,180 @@ describe("StorageApiSync", () => {
       expect(result2).toEqual(result1);
     });
   });
+
+  describe("getCoValueIDs", () => {
+    test("should return empty array when storage has no CoValues", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage({
+        storage: createSyncStorage({
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const ids = await new Promise<{ id: RawCoID }[]>((resolve) => {
+        storage.getCoValueIDs(100, 0, resolve);
+      });
+
+      expect(ids).toEqual([]);
+    });
+
+    test("should return CoValue IDs in batch after storing CoValues", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      fixtures.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const client = setupTestNode();
+      const { storage } = client.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      // Create CoValues and sync to storage
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      const map = group.createMap();
+      map.set("key", "value", "trusting");
+      await map.core.waitForSync();
+
+      const ids = await new Promise<{ id: RawCoID }[]>((resolve) => {
+        storage.getCoValueIDs(100, 0, resolve);
+      });
+
+      expect(ids.map((e) => e.id)).toContain(group.id);
+      expect(ids.map((e) => e.id)).toContain(map.id);
+      expect(ids.length).toEqual(2);
+    });
+
+    test("should paginate when there are more CoValues than the limit and return each ID only once", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      fixtures.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const client = setupTestNode();
+      const { storage } = client.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      // Create more CoValues than the page size (1 group + 4 maps = 5 CoValues, limit = 2)
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      const expectedIds = new Set<RawCoID>([group.id]);
+      const maps: ReturnType<typeof group.createMap>[] = [];
+      for (let i = 0; i < 4; i++) {
+        const map = group.createMap();
+        map.set(`key${i}`, `value${i}`, "trusting");
+        maps.push(map);
+        expectedIds.add(map.id);
+      }
+      await maps[maps.length - 1]!.core.waitForSync();
+
+      const limit = 2;
+      const allIds: RawCoID[] = [];
+      await new Promise<void>((resolve) => {
+        const fetchBatch = (offset: number) => {
+          storage.getCoValueIDs(limit, offset, (batch) => {
+            for (const { id } of batch) {
+              allIds.push(id);
+            }
+            if (batch.length >= limit) {
+              fetchBatch(offset + batch.length);
+            } else {
+              resolve();
+            }
+          });
+        };
+        fetchBatch(0);
+      });
+
+      expect(allIds).toHaveLength(expectedIds.size);
+      const seen = new Set<RawCoID>();
+      for (const id of allIds) {
+        expect(seen.has(id)).toBe(false);
+        seen.add(id);
+        expect(expectedIds.has(id)).toBe(true);
+      }
+    });
+  });
+
+  describe("getCoValueCount", () => {
+    test("should return 0 when storage has no CoValues", async () => {
+      const client = setupTestNode();
+      const { storage } = client.addStorage({
+        storage: createSyncStorage({
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const count = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+
+      expect(count).toBe(0);
+    });
+
+    test("should return CoValue count after storing CoValues", async () => {
+      const dbPath = getDbPath();
+      const fixtures = setupTestNode();
+      fixtures.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const client = setupTestNode();
+      const { storage } = client.addStorage({
+        storage: createSyncStorage({
+          filename: dbPath,
+          nodeName: "test",
+          storageName: "test-storage",
+        }),
+      });
+
+      const countEmpty = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countEmpty).toBe(0);
+
+      const group = fixtures.node.createGroup();
+      group.addMember("everyone", "reader");
+      await group.core.waitForSync();
+
+      const countOne = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countOne).toBe(1);
+
+      const map = group.createMap();
+      map.set("key", "value", "trusting");
+      await map.core.waitForSync();
+
+      const countTwo = await new Promise<number>((resolve) => {
+        storage.getCoValueCount(resolve);
+      });
+      expect(countTwo).toBe(2);
+    });
+  });
 });