cojson 0.19.19 → 0.19.21

package/src/sync.ts

@@ -3,6 +3,7 @@ import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { UnsyncedCoValuesTracker } from "./UnsyncedCoValuesTracker.js";
+import { SYNC_SCHEDULER_CONFIG } from "./config.js";
 import {
   getContenDebugInfo,
   getNewTransactionsFromContentMessage,
@@ -19,10 +20,12 @@ import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
+import type { StorageStreamingQueue } from "./queue/StorageStreamingQueue.js";
 import {
   CoValueKnownState,
   knownStateFrom,
   KnownStateSessions,
+  peerHasAllContent,
 } from "./knownState.js";
 import { StorageAPI } from "./storage/index.js";
 
@@ -426,17 +429,87 @@ export class SyncManager {
     }
   }
 
-  messagesQueue = new IncomingMessagesQueue();
+  messagesQueue = new IncomingMessagesQueue(() => this.processQueues());
+  private processing = false;
+
   pushMessage(incoming: SyncMessage, peer: PeerState) {
     this.messagesQueue.push(incoming, peer);
+  }
 
-    if (this.messagesQueue.processing) {
+  /**
+   * Get the storage streaming queue if available.
+   * Returns undefined if storage doesn't have a streaming queue.
+   */
+  private getStorageStreamingQueue(): StorageStreamingQueue | undefined {
+    const storage = this.local.storage;
+    if (storage && "streamingQueue" in storage) {
+      return storage.streamingQueue as StorageStreamingQueue;
+    }
+    return undefined;
+  }
+
+  /**
+   * Unified queue processing that coordinates both incoming messages
+   * and storage streaming entries.
+   *
+   * Processes items from both queues with priority ordering:
+   * - Incoming messages are processed via round-robin across peers
+   * - Storage streaming entries are processed by priority (MEDIUM before LOW)
+   *
+   * Implements time budget scheduling to avoid blocking the main thread.
+   */
+  private async processQueues() {
+    if (this.processing) {
       return;
     }
 
-    this.messagesQueue.processQueue((msg, peer) => {
-      this.handleSyncMessage(msg, peer);
-    });
+    this.processing = true;
+    let lastTimer = performance.now();
+
+    const streamingQueue = this.getStorageStreamingQueue();
+
+    while (true) {
+      // First, try to pull from incoming messages queue
+      const messageEntry = this.messagesQueue.pull();
+      if (messageEntry) {
+        try {
+          this.handleSyncMessage(messageEntry.msg, messageEntry.peer);
+        } catch (err) {
+          logger.error("Error processing message", { err });
+        }
+      }
+
+      // Then, try to pull from storage streaming queue
+      const pushStreamingContent = streamingQueue?.pull();
+      if (pushStreamingContent) {
+        try {
+          // Invoke the pushContent callback to stream the content
+          pushStreamingContent();
+        } catch (err) {
+          logger.error("Error processing storage streaming entry", {
+            err,
+          });
+        }
+      }
+
+      // If both queues are empty, we're done
+      if (!messageEntry && !pushStreamingContent) {
+        break;
+      }
+
+      // Check if we have blocked the main thread for too long
+      // and if so, yield to the event loop
+      const currentTimer = performance.now();
+      if (
+        currentTimer - lastTimer >
+        SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET
+      ) {
+        await new Promise<void>((resolve) => setTimeout(resolve));
+        lastTimer = performance.now();
+      }
+    }
+
+    this.processing = false;
   }
 
   addPeer(peer: Peer, skipReconciliation: boolean = false) {
@@ -511,33 +584,125 @@ export class SyncManager {
     peer.setKnownState(msg.id, knownStateFrom(msg));
     const coValue = this.local.getCoValue(msg.id);
 
+    // Fast path: CoValue is already in memory
     if (coValue.isAvailable()) {
       this.sendNewContent(msg.id, peer);
       return;
     }
 
-    const peers = this.getServerPeers(msg.id, peer.id);
+    const peerKnownState = peer.getOptimisticKnownState(msg.id);
+
+    // Fast path: Peer has no content at all - skip lazy load check, just load directly
+    if (!peerKnownState?.header) {
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+      return;
+    }
+
+    // Check storage knownState before doing full load (lazy load optimization)
+    coValue.getKnownStateFromStorage((storageKnownState) => {
+      // Race condition: CoValue might have been loaded while we were waiting for storage
+      if (coValue.isAvailable()) {
+        this.sendNewContent(msg.id, peer);
+        return;
+      }
+
+      if (!storageKnownState) {
+        // Not in storage, try loading from peers
+        this.loadFromPeersAndRespond(msg.id, peer, coValue);
+        return;
+      }
+
+      // Check if peer already has all content
+      if (peerHasAllContent(storageKnownState, peerKnownState)) {
+        // Peer already has everything - reply with known message, no full load needed
+        peer.trackToldKnownState(msg.id);
+        this.trySendToPeer(peer, {
+          action: "known",
+          ...storageKnownState,
+        });
+        return;
+      }
 
-    coValue.load(peers);
+      // Peer needs content - do full load from storage
+      this.loadFromStorageAndRespond(msg.id, peer, coValue);
+    });
+  }
+
+  /**
+   * Helper to load from storage and respond appropriately.
+   * Falls back to peers if not found in storage.
+   */
+  private loadFromStorageAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    coValue.loadFromStorage((found) => {
+      if (found && coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
+      } else {
+        this.loadFromPeersAndRespond(id, peer, coValue);
+      }
+    });
+  }
+
+  /**
+   * Helper to load from peers and respond appropriately.
+   */
+  private loadFromPeersAndRespond(
+    id: RawCoID,
+    peer: PeerState,
+    coValue: CoValueCore,
+  ) {
+    const peers = this.getServerPeers(id, peer.id);
+    coValue.loadFromPeers(peers);
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
+        this.sendNewContent(id, peer);
         return;
       }
+      this.handleLoadNotFound(id, peer);
+    };
 
-      peer.trackToldKnownState(msg.id);
+    if (peers.length > 0) {
+      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
+    } else {
+      handleLoadResult();
+    }
+  }
+
+  /**
+   * Handle case when CoValue is not found.
+   */
+  private handleLoadNotFound(id: RawCoID, peer: PeerState) {
+    peer.trackToldKnownState(id);
+    this.trySendToPeer(peer, {
+      action: "known",
+      id,
+      header: false,
+      sessions: {},
+    });
+  }
+
+  /**
+   * Request full content from a peer when we don't have the CoValue.
+   */
+  private requestFullContent(id: RawCoID, peer: PeerState | undefined) {
+    if (peer) {
       this.trySendToPeer(peer, {
         action: "known",
-        id: msg.id,
+        isCorrection: true,
+        id,
         header: false,
         sessions: {},
       });
-    };
-
-    if (peers.length > 0 || this.local.storage) {
-      coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
     } else {
-      handleLoadResult();
+      // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
+      // Should never happen
+      logger.error("Received new content with no header on a missing CoValue", {
+        id,
+      });
     }
   }
 
@@ -622,46 +787,37 @@ export class SyncManager {
      */
     if (!coValue.hasVerifiedContent()) {
       /**
-       * The peer has assumed we already have the CoValue
+       * The peer/import has assumed we already have the CoValue
        */
       if (!msg.header) {
-        // We check if the covalue was in memory and has been garbage collected
-        // In that case we should have it tracked in the storage
-        const storageKnownState = this.local.storage?.getKnownState(msg.id);
-
-        if (storageKnownState?.header) {
-          // If the CoValue has been garbage collected, we load it from the storage before handling the new content
-          coValue.loadFromStorage((found) => {
-            if (found) {
-              this.handleNewContent(msg, from);
-            } else {
-              logger.error("Known CoValue not found in storage", {
-                id: msg.id,
-              });
-            }
-          });
-          return;
-        }
-
-        // The peer assumption is not correct, so we ask for the full CoValue
-        if (peer) {
-          this.trySendToPeer(peer, {
-            action: "known",
-            isCorrection: true,
-            id: msg.id,
-            header: false,
-            sessions: {},
-          });
-        } else {
-          // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
-          // Should never happen
-          logger.error(
-            "Received new content with no header on a missing CoValue",
+        // Content from storage without header - this can happen if:
+        // 1. Storage is streaming a large CoValue in chunks
+        // 2. Server is under heavy load, so a chunk isn't processed for a long time
+        // 3. GC cleanup unmounts the CoValue while streaming is in progress
+        // 4. The chunk is finally processed, but the CoValue is no longer available
+        // TODO: Fix this by either not unmounting CoValues with active streaming,
+        // or by cleaning up the streaming queue on unmount
+        if (from === "storage") {
+          logger.warn(
+            "Received content from storage without header - CoValue may have been garbage collected mid-stream",
             {
               id: msg.id,
+              from,
             },
           );
+          return;
         }
+
+        // Try to load from storage - the CoValue might have been garbage collected from memory
+        coValue.loadFromStorage((found) => {
+          if (found) {
+            // CoValue was in storage, process the new content
+            this.handleNewContent(msg, from);
+          } else {
+            // CoValue not in storage, ask peer for full content
+            this.requestFullContent(msg.id, peer);
+          }
+        });
         return;
       }
 
@@ -1053,6 +1209,13 @@ export class SyncManager {
 
   setStorage(storage: StorageAPI) {
     this.unsyncedTracker.setStorage(storage);
+
+    const storageStreamingQueue = this.getStorageStreamingQueue();
+    if (storageStreamingQueue) {
+      storageStreamingQueue.setListener(() => {
+        this.processQueues();
+      });
+    }
   }
 
   removeStorage() {

package/src/tests/IncomingMessagesQueue.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { afterEach, describe, expect, test, vi } from "vitest";
 import { PeerState } from "../PeerState.js";
 import { IncomingMessagesQueue } from "../queue/IncomingMessagesQueue.js";
 import { ConnectedPeerChannel } from "../streamUtils.js";
@@ -8,13 +8,6 @@ import {
   tearDownTestMetricReader,
 } from "./testUtils.js";
 
-// Mock performance.now for consistent timing tests
-let mockPerformanceNow = vi.spyOn(performance, "now");
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
 function createMockPeer(id: string): Peer {
   return {
     id,
@@ -60,18 +53,15 @@ function createMockSyncMessage(
 }
 
 function setup() {
+  const processQueues = vi.fn();
   const metricReader = createTestMetricReader();
-  const queue = new IncomingMessagesQueue();
+  const queue = new IncomingMessagesQueue(processQueues);
   const peer1 = createMockPeerState("peer1");
   const peer2 = createMockPeerState("peer2");
 
-  return { queue, peer1, peer2, metricReader };
+  return { queue, peer1, peer2, metricReader, processQueues };
 }
 
-beforeEach(() => {
-  mockPerformanceNow.mockReturnValue(0);
-});
-
 afterEach(() => {
   tearDownTestMetricReader();
 });
@@ -82,7 +72,6 @@ describe("IncomingMessagesQueue", () => {
       const { queue } = setup();
       expect(queue["queues"]).toEqual([]);
       expect(queue.currentQueue).toBe(0);
-      expect(queue.processing).toBe(false);
     });
   });
 
@@ -217,142 +206,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("processQueue", () => {
-    test("should process all messages in queue", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-      ]);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should set processing flag during execution", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      let processingFlagDuringExecution = false;
-      const processingPromise = queue.processQueue(() => {
-        processingFlagDuringExecution = queue.processing;
-      });
-
-      await processingPromise;
-      expect(processingFlagDuringExecution).toBe(true);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should handle empty queue", async () => {
-      const { queue } = setup();
-      const callback = vi.fn();
-
-      await queue.processQueue(callback);
-
-      expect(callback).not.toHaveBeenCalled();
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should yield to event loop when processing takes too long", async () => {
-      const { queue, peer1 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      // Mock timing to simulate long processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(60); // After first message (60ms > 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 0);
-    });
-
-    test("should not yield to event loop when processing is fast", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      // Mock timing to simulate fast processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(30); // After message (30ms < 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).not.toHaveBeenCalled();
-    });
-
-    test("should handle callback errors gracefully", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const error = new Error("Callback error");
-
-      await queue.processQueue(() => {
-        throw error;
-      });
-
-      // The processing flag should be reset even when an error occurs
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should process messages in correct round-robin order", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-      const msg4 = createMockSyncMessage("test4");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-      queue.push(msg4, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      // Should process in round-robin: peer1, peer2, peer1, peer2
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-        { msg: msg4, peer: peer2 },
-      ]);
-    });
-  });
-
   describe("edge cases", () => {
     test("should handle peer with multiple messages correctly", () => {
       const { queue, peer1 } = setup();
@@ -411,35 +264,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("concurrent operations", () => {
-    test("should prevent multiple concurrent processQueue calls", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const firstProcessSpy = vi.fn();
-
-      const firstProcess = queue.processQueue((msg, peer) => {
-        firstProcessSpy(msg, peer);
-      });
-
-      const secondProcessSpy = vi.fn();
-
-      // Second process should not interfere
-      const secondProcess = queue.processQueue(() => {
-        secondProcessSpy();
-      });
-
-      await firstProcess;
-      await secondProcess;
-
-      expect(firstProcessSpy).toHaveBeenCalled();
-      expect(secondProcessSpy).not.toHaveBeenCalled();
-
-      expect(queue.processing).toBe(false);
-    });
-  });
-
   describe("metrics", () => {
     test("should increment push counter when pushing messages", async () => {
       const { queue, peer1, metricReader } = setup();
@@ -594,31 +418,5 @@ describe("IncomingMessagesQueue", () => {
       expect(clientPullValue).toBe(0);
       expect(serverPullValue).toBe(0);
     });
-
-    test("should track metrics during processQueue execution", async () => {
-      const { queue, peer1, metricReader } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      await queue.processQueue(() => {
-        // Process messages
-      });
-
-      const pushValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pushed",
-        { peerRole: "client" },
-      );
-
-      const pullValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pulled",
-        { peerRole: "client" },
-      );
-
-      expect(pushValue).toBe(2);
-      expect(pullValue).toBe(2);
-    });
   });
 });