cojson 0.19.18 → 0.19.20

package/src/sync.ts

@@ -2,6 +2,8 @@ import { md5 } from "@noble/hashes/legacy";
 import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
+import { UnsyncedCoValuesTracker } from "./UnsyncedCoValuesTracker.js";
+import { SYNC_SCHEDULER_CONFIG } from "./config.js";
 import {
   getContenDebugInfo,
   getNewTransactionsFromContentMessage,
@@ -18,11 +20,13 @@ import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
+import type { StorageStreamingQueue } from "./queue/StorageStreamingQueue.js";
 import {
   CoValueKnownState,
   knownStateFrom,
   KnownStateSessions,
 } from "./knownState.js";
+import { StorageAPI } from "./storage/index.js";
 
 export type SyncMessage =
   | LoadMessage
@@ -63,6 +67,15 @@ export type DoneMessage = {
   id: RawCoID;
 };
 
+/**
+ * Determines when network sync is enabled.
+ * - "always": sync is enabled for both Anonymous Authentication and Authenticated Account
+ * - "signedUp": sync is enabled when the user is authenticated
+ * - "never": sync is disabled, content stays local
+ * Can be dynamically modified to control sync behavior at runtime.
+ */
+export type SyncWhen = "always" | "signedUp" | "never";
+
 export type PeerID = string;
 
 export type DisconnectedError = "Disconnected";
@@ -121,6 +134,7 @@ export class SyncManager {
   constructor(local: LocalNode) {
     this.local = local;
     this.syncState = new SyncStateManager(this);
+    this.unsyncedTracker = new UnsyncedCoValuesTracker();
 
     this.transactionsSizeHistogram = metrics
       .getMeter("cojson")
@@ -132,6 +146,7 @@ export class SyncManager {
   }
 
   syncState: SyncStateManager;
+  unsyncedTracker: UnsyncedCoValuesTracker;
 
   disableTransactionVerification() {
     this.skipVerify = true;
@@ -154,6 +169,10 @@ export class SyncManager {
       : serverPeers;
   }
 
+  getPersistentServerPeers(id: RawCoID): PeerState[] {
+    return this.getServerPeers(id).filter((peer) => peer.persistent);
+  }
+
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
     if (!isRawCoID(msg.id)) {
       const errorType = msg.id ? "invalid" : "undefined";
@@ -259,7 +278,88 @@ export class SyncManager {
     }
   }
 
+  async resumeUnsyncedCoValues(): Promise<void> {
+    if (!this.local.storage) {
+      // No storage available, skip resumption
+      return;
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      // Load all persisted unsynced CoValues from storage
+      this.local.storage?.getUnsyncedCoValueIDs((unsyncedCoValueIDs) => {
+        const coValuesToLoad = unsyncedCoValueIDs.filter(
+          (coValueId) => !this.local.hasCoValue(coValueId),
+        );
+        if (coValuesToLoad.length === 0) {
+          resolve();
+          return;
+        }
+
+        const BATCH_SIZE = 10;
+        let processed = 0;
+
+        const processBatch = async () => {
+          const batch = coValuesToLoad.slice(processed, processed + BATCH_SIZE);
+
+          await Promise.all(
+            batch.map(
+              async (coValueId) =>
+                new Promise<void>((resolve) => {
+                  try {
+                    // Clear previous tracking (as it may include outdated peers)
+                    this.local.storage?.stopTrackingSyncState(coValueId);
+
+                    // Resume tracking sync state for this CoValue
+                    // This will add it back to the tracker and set up subscriptions
+                    this.trackSyncState(coValueId);
+
+                    // Load the CoValue from storage (this will trigger sync if peers are connected)
+                    const coValue = this.local.getCoValue(coValueId);
+                    coValue.loadFromStorage((found) => {
+                      if (!found) {
+                        // CoValue could not be loaded from storage, stop tracking
+                        this.unsyncedTracker.removeAll(coValueId);
+                      }
+                      resolve();
+                    });
+                  } catch (error) {
+                    // Handle errors gracefully - log but don't fail the entire resumption
+                    logger.warn(
+                      `Failed to resume sync for CoValue ${coValueId}:`,
+                      {
+                        err: error,
+                        coValueId,
+                      },
+                    );
+                    this.unsyncedTracker.removeAll(coValueId);
+                    resolve();
+                  }
+                }),
+            ),
+          );
+
+          processed += batch.length;
+
+          if (processed < coValuesToLoad.length) {
+            processBatch().catch(reject);
+          } else {
+            resolve();
+          }
+        };
+
+        processBatch().catch(reject);
+      });
+    });
+  }
+
   startPeerReconciliation(peer: PeerState) {
+    if (peer.role === "server" && peer.persistent) {
+      // Resume syncing unsynced CoValues asynchronously
+      this.resumeUnsyncedCoValues().catch((error) => {
+        logger.warn("Failed to resume unsynced CoValues:", error);
+      });
+    }
+
     const coValuesOrderedByDependency: CoValueCore[] = [];
 
     const seen = new Set<string>();
@@ -328,17 +428,87 @@ export class SyncManager {
     }
   }
 
-  messagesQueue = new IncomingMessagesQueue();
+  messagesQueue = new IncomingMessagesQueue(() => this.processQueues());
+  private processing = false;
+
   pushMessage(incoming: SyncMessage, peer: PeerState) {
     this.messagesQueue.push(incoming, peer);
+  }
+
+  /**
+   * Get the storage streaming queue if available.
+   * Returns undefined if storage doesn't have a streaming queue.
+   */
+  private getStorageStreamingQueue(): StorageStreamingQueue | undefined {
+    const storage = this.local.storage;
+    if (storage && "streamingQueue" in storage) {
+      return storage.streamingQueue as StorageStreamingQueue;
+    }
+    return undefined;
+  }
 
-    if (this.messagesQueue.processing) {
+  /**
+   * Unified queue processing that coordinates both incoming messages
+   * and storage streaming entries.
+   *
+   * Processes items from both queues with priority ordering:
+   * - Incoming messages are processed via round-robin across peers
+   * - Storage streaming entries are processed by priority (MEDIUM before LOW)
+   *
+   * Implements time budget scheduling to avoid blocking the main thread.
+   */
+  private async processQueues() {
+    if (this.processing) {
       return;
     }
 
-    this.messagesQueue.processQueue((msg, peer) => {
-      this.handleSyncMessage(msg, peer);
-    });
+    this.processing = true;
+    let lastTimer = performance.now();
+
+    const streamingQueue = this.getStorageStreamingQueue();
+
+    while (true) {
+      // First, try to pull from incoming messages queue
+      const messageEntry = this.messagesQueue.pull();
+      if (messageEntry) {
+        try {
+          this.handleSyncMessage(messageEntry.msg, messageEntry.peer);
+        } catch (err) {
+          logger.error("Error processing message", { err });
+        }
+      }
+
+      // Then, try to pull from storage streaming queue
+      const pushStreamingContent = streamingQueue?.pull();
+      if (pushStreamingContent) {
+        try {
+          // Invoke the pushContent callback to stream the content
+          pushStreamingContent();
+        } catch (err) {
+          logger.error("Error processing storage streaming entry", {
+            err,
+          });
+        }
+      }
+
+      // If both queues are empty, we're done
+      if (!messageEntry && !pushStreamingContent) {
+        break;
+      }
+
+      // Check if we have blocked the main thread for too long
+      // and if so, yield to the event loop
+      const currentTimer = performance.now();
+      if (
+        currentTimer - lastTimer >
+        SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET
+      ) {
+        await new Promise<void>((resolve) => setTimeout(resolve));
+        lastTimer = performance.now();
+      }
+    }
+
+    this.processing = false;
   }
 
   addPeer(peer: Peer, skipReconciliation: boolean = false) {
@@ -732,6 +902,9 @@ export class SyncManager {
 
     if (from !== "storage" && hasNewContent) {
       this.storeContent(validNewContent);
+      if (from === "import") {
+        this.trackSyncState(coValue.id);
+      }
     }
 
     for (const peer of this.getPeers(coValue.id)) {
@@ -787,6 +960,8 @@ export class SyncManager {
 
     this.storeContent(content);
 
+    this.trackSyncState(coValue.id);
+
     const contentKnownState = knownStateFromContent(content);
 
     for (const peer of this.getPeers(coValue.id)) {
@@ -811,6 +986,37 @@ export class SyncManager {
     }
   }
 
+  private trackSyncState(coValueId: RawCoID): void {
+    const peers = this.getPersistentServerPeers(coValueId);
+
+    const isSyncRequired = this.local.syncWhen !== "never";
+    if (isSyncRequired && peers.length === 0) {
+      this.unsyncedTracker.add(coValueId);
+      return;
+    }
+
+    for (const peer of peers) {
+      if (this.syncState.isSynced(peer, coValueId)) {
+        continue;
+      }
+      const alreadyTracked = this.unsyncedTracker.add(coValueId, peer.id);
+      if (alreadyTracked) {
+        continue;
+      }
+
+      const unsubscribe = this.syncState.subscribeToPeerUpdates(
+        peer.id,
+        coValueId,
+        (_knownState, syncState) => {
+          if (syncState.uploaded) {
+            this.unsyncedTracker.remove(coValueId, peer.id);
+            unsubscribe();
+          }
+        },
+      );
+    }
+  }
+
   private storeContent(content: NewContentMessage) {
     const storage = this.local.storage;
 
@@ -860,8 +1066,9 @@ export class SyncManager {
     return new Promise((resolve, reject) => {
       const unsubscribe = this.syncState.subscribeToPeerUpdates(
         peerId,
-        (knownState, syncState) => {
-          if (syncState.uploaded && knownState.id === id) {
+        id,
+        (_knownState, syncState) => {
+          if (syncState.uploaded) {
             resolve(true);
             unsubscribe?.();
             clearTimeout(timeoutId);
@@ -916,10 +1123,30 @@ export class SyncManager {
     );
   }
 
-  gracefulShutdown() {
+  setStorage(storage: StorageAPI) {
+    this.unsyncedTracker.setStorage(storage);
+
+    const storageStreamingQueue = this.getStorageStreamingQueue();
+    if (storageStreamingQueue) {
+      storageStreamingQueue.setListener(() => {
+        this.processQueues();
+      });
+    }
+  }
+
+  removeStorage() {
+    this.unsyncedTracker.removeStorage();
+  }
+
+  /**
+   * Closes all the peer connections and ensures the list of unsynced coValues is persisted to storage.
+   * @returns Promise of the current pending store operation, if any.
+   */
+  gracefulShutdown(): Promise<void> | undefined {
     for (const peer of Object.values(this.peers)) {
       peer.gracefulShutdown();
     }
+    return this.unsyncedTracker.forcePersist();
   }
 }
 

package/src/tests/IncomingMessagesQueue.test.ts

@@ -1,4 +1,4 @@
-import { afterEach, beforeEach, describe, expect, test, vi } from "vitest";
+import { afterEach, describe, expect, test, vi } from "vitest";
 import { PeerState } from "../PeerState.js";
 import { IncomingMessagesQueue } from "../queue/IncomingMessagesQueue.js";
 import { ConnectedPeerChannel } from "../streamUtils.js";
@@ -8,13 +8,6 @@ import {
   tearDownTestMetricReader,
 } from "./testUtils.js";
 
-// Mock performance.now for consistent timing tests
-let mockPerformanceNow = vi.spyOn(performance, "now");
-
-beforeEach(() => {
-  vi.clearAllMocks();
-});
-
 function createMockPeer(id: string): Peer {
   return {
     id,
@@ -60,18 +53,15 @@ function createMockSyncMessage(
 }
 
 function setup() {
+  const processQueues = vi.fn();
   const metricReader = createTestMetricReader();
-  const queue = new IncomingMessagesQueue();
+  const queue = new IncomingMessagesQueue(processQueues);
   const peer1 = createMockPeerState("peer1");
   const peer2 = createMockPeerState("peer2");
 
-  return { queue, peer1, peer2, metricReader };
+  return { queue, peer1, peer2, metricReader, processQueues };
 }
 
-beforeEach(() => {
-  mockPerformanceNow.mockReturnValue(0);
-});
-
 afterEach(() => {
   tearDownTestMetricReader();
 });
@@ -82,7 +72,6 @@ describe("IncomingMessagesQueue", () => {
       const { queue } = setup();
       expect(queue["queues"]).toEqual([]);
       expect(queue.currentQueue).toBe(0);
-      expect(queue.processing).toBe(false);
     });
   });
 
@@ -217,142 +206,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("processQueue", () => {
-    test("should process all messages in queue", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-      ]);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should set processing flag during execution", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      let processingFlagDuringExecution = false;
-      const processingPromise = queue.processQueue(() => {
-        processingFlagDuringExecution = queue.processing;
-      });
-
-      await processingPromise;
-      expect(processingFlagDuringExecution).toBe(true);
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should handle empty queue", async () => {
-      const { queue } = setup();
-      const callback = vi.fn();
-
-      await queue.processQueue(callback);
-
-      expect(callback).not.toHaveBeenCalled();
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should yield to event loop when processing takes too long", async () => {
-      const { queue, peer1 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      // Mock timing to simulate long processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(60); // After first message (60ms > 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).toHaveBeenCalledWith(expect.any(Function), 0);
-    });
-
-    test("should not yield to event loop when processing is fast", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      // Mock timing to simulate fast processing
-      mockPerformanceNow
-        .mockReturnValueOnce(0) // Initial time
-        .mockReturnValueOnce(30); // After message (30ms < 50ms threshold)
-
-      const setTimeoutSpy = vi.spyOn(global, "setTimeout");
-
-      await queue.processQueue(() => {
-        // Simulate some processing time
-      });
-
-      expect(setTimeoutSpy).not.toHaveBeenCalled();
-    });
-
-    test("should handle callback errors gracefully", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const error = new Error("Callback error");
-
-      await queue.processQueue(() => {
-        throw error;
-      });
-
-      // The processing flag should be reset even when an error occurs
-      expect(queue.processing).toBe(false);
-    });
-
-    test("should process messages in correct round-robin order", async () => {
-      const { queue, peer1, peer2 } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-      const msg3 = createMockSyncMessage("test3");
-      const msg4 = createMockSyncMessage("test4");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-      queue.push(msg3, peer2);
-      queue.push(msg4, peer2);
-
-      const processedMessages: Array<{ msg: SyncMessage; peer: PeerState }> =
-        [];
-
-      await queue.processQueue((msg, peer) => {
-        processedMessages.push({ msg, peer });
-      });
-
-      // Should process in round-robin: peer1, peer2, peer1, peer2
-      expect(processedMessages).toEqual([
-        { msg: msg1, peer: peer1 },
-        { msg: msg3, peer: peer2 },
-        { msg: msg2, peer: peer1 },
-        { msg: msg4, peer: peer2 },
-      ]);
-    });
-  });
-
   describe("edge cases", () => {
     test("should handle peer with multiple messages correctly", () => {
       const { queue, peer1 } = setup();
@@ -411,35 +264,6 @@ describe("IncomingMessagesQueue", () => {
     });
   });
 
-  describe("concurrent operations", () => {
-    test("should prevent multiple concurrent processQueue calls", async () => {
-      const { queue, peer1 } = setup();
-      const msg = createMockSyncMessage("test");
-      queue.push(msg, peer1);
-
-      const firstProcessSpy = vi.fn();
-
-      const firstProcess = queue.processQueue((msg, peer) => {
-        firstProcessSpy(msg, peer);
-      });
-
-      const secondProcessSpy = vi.fn();
-
-      // Second process should not interfere
-      const secondProcess = queue.processQueue(() => {
-        secondProcessSpy();
-      });
-
-      await firstProcess;
-      await secondProcess;
-
-      expect(firstProcessSpy).toHaveBeenCalled();
-      expect(secondProcessSpy).not.toHaveBeenCalled();
-
-      expect(queue.processing).toBe(false);
-    });
-  });
-
   describe("metrics", () => {
     test("should increment push counter when pushing messages", async () => {
       const { queue, peer1, metricReader } = setup();
@@ -594,31 +418,5 @@ describe("IncomingMessagesQueue", () => {
       expect(clientPullValue).toBe(0);
       expect(serverPullValue).toBe(0);
     });
-
-    test("should track metrics during processQueue execution", async () => {
-      const { queue, peer1, metricReader } = setup();
-      const msg1 = createMockSyncMessage("test1");
-      const msg2 = createMockSyncMessage("test2");
-
-      queue.push(msg1, peer1);
-      queue.push(msg2, peer1);
-
-      await queue.processQueue(() => {
-        // Process messages
-      });
-
-      const pushValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pushed",
-        { peerRole: "client" },
-      );
-
-      const pullValue = await metricReader.getMetricValue(
-        "jazz.messagequeue.incoming.pulled",
-        { peerRole: "client" },
-      );
-
-      expect(pushValue).toBe(2);
-      expect(pullValue).toBe(2);
-    });
   });
 });