cojson 0.19.19 → 0.19.20

package/src/sync.ts

@@ -3,6 +3,7 @@ import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { UnsyncedCoValuesTracker } from "./UnsyncedCoValuesTracker.js";
+import { SYNC_SCHEDULER_CONFIG } from "./config.js";
 import {
   getContenDebugInfo,
   getNewTransactionsFromContentMessage,
@@ -19,6 +20,7 @@ import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
+import type { StorageStreamingQueue } from "./queue/StorageStreamingQueue.js";
 import {
   CoValueKnownState,
   knownStateFrom,
@@ -426,17 +428,87 @@ export class SyncManager {
     }
   }
 
-  messagesQueue = new IncomingMessagesQueue();
+  messagesQueue = new IncomingMessagesQueue(() => this.processQueues());
+  private processing = false;
+
   pushMessage(incoming: SyncMessage, peer: PeerState) {
     this.messagesQueue.push(incoming, peer);
+  }
+
+  /**
+   * Get the storage streaming queue if available.
+   * Returns undefined if storage doesn't have a streaming queue.
+   */
+  private getStorageStreamingQueue(): StorageStreamingQueue | undefined {
+    const storage = this.local.storage;
+    if (storage && "streamingQueue" in storage) {
+      return storage.streamingQueue as StorageStreamingQueue;
+    }
+    return undefined;
+  }
 
-    if (this.messagesQueue.processing) {
+  /**
+   * Unified queue processing that coordinates both incoming messages
+   * and storage streaming entries.
+   *
+   * Processes items from both queues with priority ordering:
+   * - Incoming messages are processed via round-robin across peers
+   * - Storage streaming entries are processed by priority (MEDIUM before LOW)
+   *
+   * Implements time budget scheduling to avoid blocking the main thread.
+   */
+  private async processQueues() {
+    if (this.processing) {
       return;
     }
 
-    this.messagesQueue.processQueue((msg, peer) => {
-      this.handleSyncMessage(msg, peer);
-    });
+    this.processing = true;
+    let lastTimer = performance.now();
+
+    const streamingQueue = this.getStorageStreamingQueue();
+
+    while (true) {
+      // First, try to pull from incoming messages queue
+      const messageEntry = this.messagesQueue.pull();
+      if (messageEntry) {
+        try {
+          this.handleSyncMessage(messageEntry.msg, messageEntry.peer);
+        } catch (err) {
+          logger.error("Error processing message", { err });
+        }
+      }
+
+      // Then, try to pull from storage streaming queue
+      const pushStreamingContent = streamingQueue?.pull();
+      if (pushStreamingContent) {
+        try {
+          // Invoke the pushContent callback to stream the content
+          pushStreamingContent();
+        } catch (err) {
+          logger.error("Error processing storage streaming entry", {
+            err,
+          });
+        }
+      }
+
+      // If both queues are empty, we're done
+      if (!messageEntry && !pushStreamingContent) {
+        break;
+      }
+
+      // Check if we have blocked the main thread for too long
+      // and if so, yield to the event loop
+      const currentTimer = performance.now();
+      if (
+        currentTimer - lastTimer >
+        SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET
+      ) {
+        await new Promise<void>((resolve) => setTimeout(resolve));
+        lastTimer = performance.now();
+      }
+    }
+
+    this.processing = false;
   }
 
   addPeer(peer: Peer, skipReconciliation: boolean = false) {
@@ -1053,6 +1125,13 @@ export class SyncManager {
 
   setStorage(storage: StorageAPI) {
     this.unsyncedTracker.setStorage(storage);
+
+    const storageStreamingQueue = this.getStorageStreamingQueue();
+    if (storageStreamingQueue) {
+      storageStreamingQueue.setListener(() => {
+        this.processQueues();
+      });
+    }
   }
 
   removeStorage() {

package/src/tests/IncomingMessagesQueue.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { afterEach, describe, expect, test, vi } from "vitest";
 import { PeerState } from "../PeerState.js";
 import { IncomingMessagesQueue } from "../queue/IncomingMessagesQueue.js";
 import { ConnectedPeerChannel } from "../streamUtils.js";
@@ -8,13 +8,6 @@ import {
   tearDownTestMetricReader,
 } from "./testUtils.js";
 
-// Mock performance.now for consistent timing tests
-let mockPerformanceNow = vi.spyOn(performance, "now");
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
 function createMockPeer(id: string): Peer {
   return {
     id,
@@ -60,18 +53,15 @@ function createMockSyncMessage(
 }
 
 function setup() {
+  const processQueues = vi.fn();
   const metricReader = createTestMetricReader();
-  const queue = new IncomingMessagesQueue();
+  const queue = new IncomingMessagesQueue(processQueues);
   const peer1 = createMockPeerState("peer1");
   const peer2 = createMockPeerState("peer2");
 
-  return { queue, peer1, peer2, metricReader };
+  return { queue, peer1, peer2, metricReader, processQueues };
 }
 
-beforeEach(() => {
-  mockPerformanceNow.mockReturnValue(0);
-});
-
 afterEach(() => {
   tearDownTestMetricReader();
 });
@@ -82,7 +72,6 @@ describe("IncomingMessagesQueue", () => {
       const { queue } = setup();
       expect(queue["queues"]).toEqual([]);
       expect(queue.currentQueue).toBe(0);
-      expect(queue.processing).toBe(false);
     });
   });
 
@@ -217,142 +206,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("processQueue", () => {
-    test("should process all messages in queue", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-      ]);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should set processing flag during execution", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      let processingFlagDuringExecution = false;
-      const processingPromise = queue.processQueue(() => {
-        processingFlagDuringExecution = queue.processing;
-      });
-
-      await processingPromise;
-      expect(processingFlagDuringExecution).toBe(true);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should handle empty queue", async () => {
-      const { queue } = setup();
-      const callback = vi.fn();
-
-      await queue.processQueue(callback);
-
-      expect(callback).not.toHaveBeenCalled();
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should yield to event loop when processing takes too long", async () => {
-      const { queue, peer1 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      // Mock timing to simulate long processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(60); // After first message (60ms > 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 0);
-    });
-
-    test("should not yield to event loop when processing is fast", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      // Mock timing to simulate fast processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(30); // After message (30ms < 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).not.toHaveBeenCalled();
-    });
-
-    test("should handle callback errors gracefully", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const error = new Error("Callback error");
-
-      await queue.processQueue(() => {
-        throw error;
-      });
-
-      // The processing flag should be reset even when an error occurs
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should process messages in correct round-robin order", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-      const msg4 = createMockSyncMessage("test4");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-      queue.push(msg4, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      // Should process in round-robin: peer1, peer2, peer1, peer2
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-        { msg: msg4, peer: peer2 },
-      ]);
-    });
-  });
-
   describe("edge cases", () => {
     test("should handle peer with multiple messages correctly", () => {
       const { queue, peer1 } = setup();
@@ -411,35 +264,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("concurrent operations", () => {
-    test("should prevent multiple concurrent processQueue calls", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const firstProcessSpy = vi.fn();
-
-      const firstProcess = queue.processQueue((msg, peer) => {
-        firstProcessSpy(msg, peer);
-      });
-
-      const secondProcessSpy = vi.fn();
-
-      // Second process should not interfere
-      const secondProcess = queue.processQueue(() => {
-        secondProcessSpy();
-      });
-
-      await firstProcess;
-      await secondProcess;
-
-      expect(firstProcessSpy).toHaveBeenCalled();
-      expect(secondProcessSpy).not.toHaveBeenCalled();
-
-      expect(queue.processing).toBe(false);
-    });
-  });
-
   describe("metrics", () => {
     test("should increment push counter when pushing messages", async () => {
       const { queue, peer1, metricReader } = setup();
@@ -594,31 +418,5 @@ describe("IncomingMessagesQueue", () => {
       expect(clientPullValue).toBe(0);
       expect(serverPullValue).toBe(0);
     });
-
-    test("should track metrics during processQueue execution", async () => {
-      const { queue, peer1, metricReader } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      await queue.processQueue(() => {
-        // Process messages
-      });
-
-      const pushValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pushed",
-        { peerRole: "client" },
-      );
-
-      const pullValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pulled",
-        { peerRole: "client" },
-      );
-
-      expect(pushValue).toBe(2);
-      expect(pullValue).toBe(2);
-    });
   });
 });

package/src/tests/StorageStreamingQueue.test.ts

@@ -0,0 +1,276 @@
+import { describe, expect, test, vi } from "vitest";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
+
+describe("StorageStreamingQueue", () => {
+  describe("constructor", () => {
+    test("should initialize with empty queues", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.isEmpty()).toBe(true);
+    });
+  });
+
+  describe("push", () => {
+    test("should add MEDIUM priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should add LOW priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should add HIGH priority entry to queue", () => {
+      const queue = new StorageStreamingQueue();
+
+      queue.push(() => {}, CO_VALUE_PRIORITY.HIGH);
+
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should accept multiple entries", () => {
+      const queue = new StorageStreamingQueue();
+      const entry1 = () => {};
+      const entry2 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry2, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.pull()).toBe(entry1);
+      expect(queue.pull()).toBe(entry2);
+    });
+  });
+
+  describe("pull", () => {
+    test("should return undefined for empty queue", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.pull()).toBeUndefined();
+    });
+
+    test("should return and remove entry from queue", () => {
+      const queue = new StorageStreamingQueue();
+      const entry = () => {};
+
+      queue.push(entry, CO_VALUE_PRIORITY.MEDIUM);
+      const pulled = queue.pull();
+
+      expect(pulled).toBe(entry);
+      expect(queue.isEmpty()).toBe(true);
+    });
+
+    test("should pull HIGH priority before MEDIUM and LOW", () => {
+      const queue = new StorageStreamingQueue();
+      const lowEntry = () => {};
+      const mediumEntry = () => {};
+      const highEntry = () => {};
+
+      // Push in reverse order
+      queue.push(lowEntry, CO_VALUE_PRIORITY.LOW);
+      queue.push(mediumEntry, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(highEntry, CO_VALUE_PRIORITY.HIGH);
+
+      // Should pull HIGH first, then MEDIUM, then LOW
+      expect(queue.pull()).toBe(highEntry);
+      expect(queue.pull()).toBe(mediumEntry);
+      expect(queue.pull()).toBe(lowEntry);
+    });
+
+    test("should pull MEDIUM priority before LOW priority", () => {
+      const queue = new StorageStreamingQueue();
+      const lowEntry = () => {};
+      const mediumEntry = () => {};
+
+      // Push LOW first, then MEDIUM
+      queue.push(lowEntry, CO_VALUE_PRIORITY.LOW);
+      queue.push(mediumEntry, CO_VALUE_PRIORITY.MEDIUM);
+
+      // Should pull MEDIUM first
+      expect(queue.pull()).toBe(mediumEntry);
+      expect(queue.pull()).toBe(lowEntry);
+    });
+
+    test("should pull entries in FIFO order within same priority", () => {
+      const queue = new StorageStreamingQueue();
+      const entry1 = () => {};
+      const entry2 = () => {};
+      const entry3 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry2, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(entry3, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(queue.pull()).toBe(entry1);
+      expect(queue.pull()).toBe(entry2);
+      expect(queue.pull()).toBe(entry3);
+    });
+
+    test("should handle interleaved priorities correctly", () => {
+      const queue = new StorageStreamingQueue();
+      const low1 = () => {};
+      const medium1 = () => {};
+      const high1 = () => {};
+      const low2 = () => {};
+      const medium2 = () => {};
+      const high2 = () => {};
+
+      queue.push(low1, CO_VALUE_PRIORITY.LOW);
+      queue.push(medium1, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(high1, CO_VALUE_PRIORITY.HIGH);
+      queue.push(low2, CO_VALUE_PRIORITY.LOW);
+      queue.push(medium2, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(high2, CO_VALUE_PRIORITY.HIGH);
+
+      // All HIGH should come first, in order
+      expect(queue.pull()).toBe(high1);
+      expect(queue.pull()).toBe(high2);
+      // Then all MEDIUM, in order
+      expect(queue.pull()).toBe(medium1);
+      expect(queue.pull()).toBe(medium2);
+      // Then all LOW, in order
+      expect(queue.pull()).toBe(low1);
+      expect(queue.pull()).toBe(low2);
+    });
+  });
+
+  describe("isEmpty", () => {
+    test("should return true for empty queue", () => {
+      const queue = new StorageStreamingQueue();
+      expect(queue.isEmpty()).toBe(true);
+    });
+
+    test("should return false when MEDIUM queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return false when LOW queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return false when HIGH queue has entries", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.HIGH);
+      expect(queue.isEmpty()).toBe(false);
+    });
+
+    test("should return true after all entries are pulled", () => {
+      const queue = new StorageStreamingQueue();
+      queue.push(() => {}, CO_VALUE_PRIORITY.MEDIUM);
+      queue.push(() => {}, CO_VALUE_PRIORITY.LOW);
+
+      queue.pull();
+      queue.pull();
+
+      expect(queue.isEmpty()).toBe(true);
+    });
+  });
+
+  describe("callback invocation", () => {
+    test("should not invoke callback when pushed", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    test("should not invoke callback when pulled", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      queue.pull();
+
+      expect(callback).not.toHaveBeenCalled();
+    });
+
+    test("should allow caller to invoke callback after pull", () => {
+      const queue = new StorageStreamingQueue();
+      const callback = vi.fn();
+
+      queue.push(callback, CO_VALUE_PRIORITY.MEDIUM);
+      const pulled = queue.pull();
+
+      expect(callback).not.toHaveBeenCalled();
+
+      pulled?.();
+
+      expect(callback).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("setListener and emit", () => {
+    test("should call listener when emit is called", () => {
+      const queue = new StorageStreamingQueue();
+      const listener = vi.fn();
+
+      queue.setListener(listener);
+      queue.emit();
+
+      expect(listener).toHaveBeenCalledTimes(1);
+    });
+
+    test("should not throw when emit is called without listener", () => {
+      const queue = new StorageStreamingQueue();
+
+      expect(() => queue.emit()).not.toThrow();
+    });
+
+    test("should call listener multiple times on multiple emits", () => {
+      const queue = new StorageStreamingQueue();
+      const listener = vi.fn();
+
+      queue.setListener(listener);
+      queue.emit();
+      queue.emit();
+      queue.emit();
+
+      expect(listener).toHaveBeenCalledTimes(3);
+    });
+
+    test("should use latest listener when setListener is called multiple times", () => {
+      const queue = new StorageStreamingQueue();
+      const listener1 = vi.fn();
+      const listener2 = vi.fn();
+
+      queue.setListener(listener1);
+      queue.setListener(listener2);
+      queue.emit();
+
+      expect(listener1).not.toHaveBeenCalled();
+      expect(listener2).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("edge cases", () => {
+    test("should handle alternating push and pull operations", () => {
+      const queue = new StorageStreamingQueue();
+
+      const entry1 = () => {};
+      const entry2 = () => {};
+      const entry3 = () => {};
+
+      queue.push(entry1, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.pull()).toBe(entry1);
+
+      queue.push(entry2, CO_VALUE_PRIORITY.LOW);
+      expect(queue.pull()).toBe(entry2);
+
+      expect(queue.isEmpty()).toBe(true);
+
+      queue.push(entry3, CO_VALUE_PRIORITY.MEDIUM);
+      expect(queue.pull()).toBe(entry3);
+    });
+  });
+});