cojson 0.19.18 → 0.19.20

package/src/UnsyncedCoValuesTracker.ts

@@ -0,0 +1,272 @@
+import type { RawCoID } from "./ids.js";
+import { logger } from "./logger.js";
+import type { PeerID } from "./sync.js";
+import type { StorageAPI } from "./storage/types.js";
+
+/**
+ * Used to track a CoValue that hasn't been synced to any peer,
+ * because none is currently connected.
+ */
+const ANY_PEER_ID: PeerID = "any";
+
+// Flush pending updates to storage after 200ms
+let BATCH_DELAY_MS = 200;
+
+/**
+ * Set the delay for flushing pending sync state updates to storage.
+ * @internal
+ */
+export function setSyncStateTrackingBatchDelay(delay: number): void {
+  BATCH_DELAY_MS = delay;
+}
+
+type PendingUpdate = {
+  id: RawCoID;
+  peerId: PeerID;
+  synced: boolean;
+};
+
+/**
+ * Tracks CoValues that have unsynced changes to specific peers.
+ * Maintains an in-memory map and periodically persists to storage.
+ */
+export class UnsyncedCoValuesTracker {
+  private unsynced: Map<RawCoID, Set<PeerID>> = new Map();
+  private coValueListeners: Map<RawCoID, Set<(synced: boolean) => void>> =
+    new Map();
+  // Listeners for global "all synced" status changes
+  private globalListeners: Set<(synced: boolean) => void> = new Set();
+
+  // Pending updates to be persisted
+  private pendingUpdates: PendingUpdate[] = [];
+  private flushTimer: ReturnType<typeof setTimeout> | undefined;
+
+  private storage?: StorageAPI;
+
+  /**
+   * Add a CoValue as unsynced to a specific peer.
+   * Triggers persistence if storage is available.
+   * @returns true if the CoValue was already tracked, false otherwise.
+   */
+  add(id: RawCoID, peerId: PeerID = ANY_PEER_ID): boolean {
+    if (!this.unsynced.has(id)) {
+      this.unsynced.set(id, new Set());
+    }
+    const peerSet = this.unsynced.get(id)!;
+
+    const alreadyTracked = peerSet.has(peerId);
+    if (!alreadyTracked) {
+      // Only update if this is a new peer
+      peerSet.add(peerId);
+
+      this.schedulePersist(id, peerId, false);
+
+      this.notifyCoValueListeners(id, false);
+      this.notifyGlobalListeners(false);
+    }
+
+    return alreadyTracked;
+  }
+
+  /**
+   * Remove a CoValue from being unsynced to a specific peer.
+   * Triggers persistence if storage is available.
+   */
+  remove(id: RawCoID, peerId: PeerID = ANY_PEER_ID): void {
+    const peerSet = this.unsynced.get(id);
+    if (!peerSet || !peerSet.has(peerId)) {
+      return;
+    }
+
+    peerSet.delete(peerId);
+
+    // If no more unsynced peers for this CoValue, remove the entry
+    if (peerSet.size === 0) {
+      this.unsynced.delete(id);
+    }
+
+    this.schedulePersist(id, peerId, true);
+
+    const isSynced = !this.unsynced.has(id);
+    this.notifyCoValueListeners(id, isSynced);
+    this.notifyGlobalListeners(this.isAllSynced());
+  }
+
+  /**
+   * Remove all tracking for a CoValue (all peers).
+   * Triggers persistence if storage is available.
+   */
+  removeAll(id: RawCoID): void {
+    const peerSet = this.unsynced.get(id);
+    if (!peerSet) {
+      return;
+    }
+
+    // Remove all peers for this CoValue
+    const peersToRemove = Array.from(peerSet);
+    for (const peerId of peersToRemove) {
+      this.remove(id, peerId);
+    }
+  }
+
+  forcePersist(): Promise<void> | undefined {
+    return this.flush();
+  }
+
+  private schedulePersist(id: RawCoID, peerId: PeerID, synced: boolean): void {
+    const storage = this.storage;
+    if (!storage) {
+      return;
+    }
+
+    this.pendingUpdates.push({ id, peerId, synced });
+    if (!this.flushTimer) {
+      this.flushTimer = setTimeout(() => {
+        this.flush();
+      }, BATCH_DELAY_MS);
+    }
+  }
+
+  /**
+   * Flush all pending persistence updates in a batch
+   */
+  private flush(): Promise<void> | undefined {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = undefined;
+    }
+
+    if (this.pendingUpdates.length === 0) {
+      return;
+    }
+
+    const storage = this.storage;
+    if (!storage) {
+      return;
+    }
+
+    const filteredUpdates = this.simplifyPendingUpdates(this.pendingUpdates);
+    this.pendingUpdates = [];
+
+    return new Promise((resolve) => {
+      try {
+        storage.trackCoValuesSyncState(filteredUpdates, () => resolve());
+      } catch (error) {
+        logger.warn("Failed to persist batched unsynced CoValue tracking", {
+          err: error,
+        });
+        resolve();
+      }
+    });
+  }
+
+  /**
+   * Get all CoValue IDs that have at least one unsynced peer.
+   */
+  getAll(): RawCoID[] {
+    return Array.from(this.unsynced.keys());
+  }
+
+  /**
+   * Check if all CoValues are synced
+   */
+  isAllSynced(): boolean {
+    return this.unsynced.size === 0;
+  }
+
+  /**
+   * Check if a specific CoValue is tracked as unsynced.
+   */
+  has(id: RawCoID): boolean {
+    return this.unsynced.has(id);
+  }
+
+  /**
+   * Subscribe to changes in whether a specific CoValue is synced.
+   * The listener is called immediately with the current state.
+   * @returns Unsubscribe function
+   */
+  subscribe(id: RawCoID, listener: (synced: boolean) => void): () => void;
+  /**
+   * Subscribe to changes in whether all CoValues are synced.
+   * The listener is called immediately with the current state.
+   * @returns Unsubscribe function
+   */
+  subscribe(listener: (synced: boolean) => void): () => void;
+  subscribe(
+    idOrListener: RawCoID | ((synced: boolean) => void),
+    listener?: (synced: boolean) => void,
+  ): () => void {
+    if (typeof idOrListener === "string" && listener) {
+      const id = idOrListener;
+      if (!this.coValueListeners.has(id)) {
+        this.coValueListeners.set(id, new Set());
+      }
+      this.coValueListeners.get(id)!.add(listener);
+
+      // Call immediately with current state
+      const isSynced = !this.unsynced.has(id);
+      listener(isSynced);
+
+      return () => {
+        const listeners = this.coValueListeners.get(id);
+        if (listeners) {
+          listeners.delete(listener);
+          if (listeners.size === 0) {
+            this.coValueListeners.delete(id);
+          }
+        }
+      };
+    }
+
+    const globalListener = idOrListener as (synced: boolean) => void;
+    this.globalListeners.add(globalListener);
+
+    // Call immediately with current state
+    globalListener(this.isAllSynced());
+
+    return () => {
+      this.globalListeners.delete(globalListener);
+    };
+  }
+
+  setStorage(storage: StorageAPI) {
+    this.storage = storage;
+  }
+
+  removeStorage() {
+    this.storage = undefined;
+  }
+
+  /**
+   * Notify all listeners for a specific CoValue about sync status change.
+   */
+  private notifyCoValueListeners(id: RawCoID, synced: boolean): void {
+    const listeners = this.coValueListeners.get(id);
+    if (listeners) {
+      for (const listener of listeners) {
+        listener(synced);
+      }
+    }
+  }
+
+  /**
+   * Notify all global listeners about "all synced" status change.
+   */
+  private notifyGlobalListeners(allSynced: boolean): void {
+    for (const listener of this.globalListeners) {
+      listener(allSynced);
+    }
+  }
+
+  /**
+   * Keep only the last update for each (id, peerId) combination
+   */
+  private simplifyPendingUpdates(updates: PendingUpdate[]): PendingUpdate[] {
+    const latestUpdates = new Map<string, PendingUpdate>();
+    for (const update of updates) {
+      latestUpdates.set(`${update.id}|${update.peerId}`, update);
+    }
+    return Array.from(latestUpdates.values());
+  }
+}

package/src/config.ts

@@ -60,3 +60,16 @@ export function setGarbageCollectorMaxAge(maxAge: number) {
 export function setGarbageCollectorInterval(interval: number) {
   GARBAGE_COLLECTOR_CONFIG.INTERVAL = interval;
 }
+
+export const WEBSOCKET_CONFIG = {
+  MAX_OUTGOING_MESSAGES_CHUNK_BYTES: 25_000,
+  OUTGOING_MESSAGES_CHUNK_DELAY: 5,
+};
+
+export function setMaxOutgoingMessagesChunkBytes(bytes: number) {
+  WEBSOCKET_CONFIG.MAX_OUTGOING_MESSAGES_CHUNK_BYTES = bytes;
+}
+
+export function setOutgoingMessagesChunkDelay(delay: number) {
+  WEBSOCKET_CONFIG.OUTGOING_MESSAGES_CHUNK_DELAY = delay;
+}

package/src/exports.ts

@@ -65,12 +65,13 @@ import type { AgentID, RawCoID, SessionID } from "./ids.js";
 import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { isAccountRole } from "./permissions.js";
-import type { Peer, SyncMessage } from "./sync.js";
+import type { Peer, SyncMessage, SyncWhen } from "./sync.js";
 import {
   DisconnectedError,
   SyncManager,
   hwrServerPeerSelector,
 } from "./sync.js";
+import { setSyncStateTrackingBatchDelay } from "./UnsyncedCoValuesTracker.js";
 import { emptyKnownState } from "./knownState.js";
 
 import {
@@ -81,10 +82,13 @@ import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import {
   CO_VALUE_LOADING_CONFIG,
   TRANSACTION_CONFIG,
+  WEBSOCKET_CONFIG,
   setCoValueLoadingMaxRetries,
   setCoValueLoadingRetryDelay,
   setCoValueLoadingTimeout,
   setIncomingMessagesTimeBudget,
+  setMaxOutgoingMessagesChunkBytes,
+  setOutgoingMessagesChunkDelay,
   setMaxRecommendedTxSize,
 } from "./config.js";
 import { LogLevel, logger } from "./logger.js";
@@ -125,6 +129,7 @@ export const cojsonInternals = {
   setCoValueLoadingRetryDelay,
   setCoValueLoadingMaxRetries,
   setCoValueLoadingTimeout,
+  setSyncStateTrackingBatchDelay,
   ConnectedPeerChannel,
   textEncoder,
   textDecoder,
@@ -133,6 +138,9 @@ export const cojsonInternals = {
   TRANSACTION_CONFIG,
   setMaxRecommendedTxSize,
   canBeBranched,
+  WEBSOCKET_CONFIG,
+  setMaxOutgoingMessagesChunkBytes,
+  setOutgoingMessagesChunkDelay,
 };
 
 export {
@@ -193,6 +201,7 @@ export type {
   AccountRole,
   AvailableCoValueCore,
   PeerState,
+  SyncWhen,
   CoValueHeader,
 };
 

package/src/localNode.ts

@@ -34,7 +34,7 @@ import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID, isRawCoID } from "./ids.js";
 import { logger } from "./logger.js";
 import { StorageAPI } from "./storage/index.js";
-import { Peer, PeerID, SyncManager } from "./sync.js";
+import { Peer, PeerID, SyncManager, type SyncWhen } from "./sync.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
@@ -75,6 +75,7 @@ export class LocalNode {
     agentSecret: AgentSecret,
     currentSessionID: SessionID,
     crypto: CryptoProvider,
+    public readonly syncWhen?: SyncWhen,
   ) {
     this.agentSecret = agentSecret;
     this.currentSessionID = currentSessionID;
@@ -94,11 +95,13 @@ export class LocalNode {
 
   setStorage(storage: StorageAPI) {
     this.storage = storage;
+    this.syncManager.setStorage(storage);
   }
 
   removeStorage() {
     this.storage?.close();
     this.storage = undefined;
+    this.syncManager.removeStorage();
   }
 
   hasCoValue(id: RawCoID) {
@@ -178,12 +181,14 @@ export class LocalNode {
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
     peers?: Peer[];
+    syncWhen?: SyncWhen;
     storage?: StorageAPI;
   }): RawAccount {
     const {
       crypto,
       initialAgentSecret = crypto.newRandomAgentSecret(),
       peers = [],
+      syncWhen,
     } = opts;
     const accountHeader = accountHeaderForInitialAgentSecret(
       initialAgentSecret,
@@ -195,6 +200,7 @@ export class LocalNode {
       initialAgentSecret,
       crypto.newRandomSessionID(accountID as RawAccountID),
       crypto,
+      syncWhen,
     );
 
     if (opts.storage) {
@@ -236,6 +242,7 @@ export class LocalNode {
   static async withNewlyCreatedAccount({
     creationProps,
     peers,
+    syncWhen,
     migration,
     crypto,
     initialAgentSecret = crypto.newRandomAgentSecret(),
@@ -243,6 +250,7 @@ export class LocalNode {
   }: {
     creationProps: { name: string };
     peers?: Peer[];
+    syncWhen?: SyncWhen;
     migration?: RawAccountMigration<AccountMeta>;
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
@@ -257,6 +265,7 @@ export class LocalNode {
       crypto,
       initialAgentSecret,
       peers,
+      syncWhen,
       storage,
     });
     const node = account.core.node;
@@ -299,6 +308,7 @@ export class LocalNode {
     accountSecret,
     sessionID,
     peers,
+    syncWhen,
     crypto,
     migration,
     storage,
@@ -307,6 +317,7 @@ export class LocalNode {
     accountSecret: AgentSecret;
     sessionID: SessionID | undefined;
     peers: Peer[];
+    syncWhen?: SyncWhen;
     crypto: CryptoProvider;
     migration?: RawAccountMigration<AccountMeta>;
     storage?: StorageAPI;
@@ -318,6 +329,7 @@ export class LocalNode {
         accountSecret,
         sessionID || crypto.newRandomSessionID(accountID),
         crypto,
+        syncWhen,
       );
 
       if (storage) {
@@ -817,9 +829,9 @@ export class LocalNode {
    *
    * @returns Promise of the current pending store operation, if any.
    */
-  gracefulShutdown(): Promise<unknown> | undefined {
-    this.syncManager.gracefulShutdown();
+  async gracefulShutdown(): Promise<unknown> {
     this.garbageCollector?.stop();
+    await this.syncManager.gracefulShutdown();
     return this.storage?.close();
   }
 }

package/src/queue/IncomingMessagesQueue.ts

@@ -1,17 +1,15 @@
 import { Counter, ValueType, metrics } from "@opentelemetry/api";
 import type { PeerState } from "../PeerState.js";
-import { SYNC_SCHEDULER_CONFIG } from "../config.js";
-import { logger } from "../logger.js";
 import type { SyncMessage } from "../sync.js";
 import { LinkedList } from "./LinkedList.js";
 
 /**
- * A queue that schedules messages across different peers using a round-robin approach.
+ * A queue that manages incoming sync messages across different peers using a round-robin approach.
  *
  * This class manages incoming sync messages from multiple peers, ensuring fair processing
- * by cycling through each peer's message queue in a round-robin fashion. It also implements
- * collaborative scheduling on message processing, pausing when the main thread is blocked
- * for more than 50ms.
+ * by cycling through each peer's message queue in a round-robin fashion.
+ *
+ * Queue processing and scheduling is handled by SyncManager.processQueues().
  */
 export class IncomingMessagesQueue {
   private pullCounter: Counter;
@@ -21,7 +19,7 @@ export class IncomingMessagesQueue {
   peerToQueue: WeakMap<PeerState, LinkedList<SyncMessage>>;
   currentQueue = 0;
 
-  constructor() {
+  constructor(private processQueues: () => void) {
     this.pullCounter = metrics
       .getMeter("cojson")
       .createCounter(`jazz.messagequeue.incoming.pulled`, {
@@ -74,6 +72,8 @@ export class IncomingMessagesQueue {
     this.pushCounter.add(1, {
       peerRole: peer.role,
     });
+
+    this.processQueues();
   }
 
   public pull() {
@@ -107,36 +107,4 @@ export class IncomingMessagesQueue {
 
     return undefined;
   }
-
-  processing = false;
-
-  async processQueue(callback: (msg: SyncMessage, peer: PeerState) => void) {
-    this.processing = true;
-
-    let entry: { msg: SyncMessage; peer: PeerState } | undefined;
-    let lastTimer = performance.now();
-
-    while ((entry = this.pull())) {
-      const { msg, peer } = entry;
-
-      try {
-        callback(msg, peer);
-      } catch (err) {
-        logger.error("Error processing message", { err });
-      }
-
-      const currentTimer = performance.now();
-
-      // We check if we have blocked the main thread for too long
-      // and if so, we schedule a timer task to yield to the event loop
-      if (
-        currentTimer - lastTimer >
-        SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET
-      ) {
-        await new Promise((resolve) => setTimeout(resolve, 0));
-      }
-    }
-
-    this.processing = false;
-  }
 }

package/src/queue/LinkedList.ts

@@ -108,7 +108,7 @@ class QueueMeter {
   }
 }
 export function meteredList<T>(
-  type: "incoming" | "outgoing",
+  type: "incoming" | "outgoing" | "storage-streaming",
   attrs?: Record<string, string | number>,
 ) {
   return new LinkedList<T>(new QueueMeter("jazz.messagequeue." + type, attrs));

package/src/queue/StorageStreamingQueue.ts

@@ -0,0 +1,96 @@
+import { CO_VALUE_PRIORITY, type CoValuePriority } from "../priority.js";
+import { LinkedList, meteredList } from "./LinkedList.js";
+
+/**
+ * A callback that pushes content when invoked.
+ * Content is only fetched from the database when this callback is called.
+ */
+export type ContentCallback = () => void;
+
+// All priorities use the queue, processed in order: HIGH > MEDIUM > LOW
+const PRIORITY_TO_QUEUE_INDEX = {
+  [CO_VALUE_PRIORITY.HIGH]: 0,
+  [CO_VALUE_PRIORITY.MEDIUM]: 1,
+  [CO_VALUE_PRIORITY.LOW]: 2,
+} as const;
+
+type StreamingQueueTuple = [
+  high: LinkedList<ContentCallback>,
+  medium: LinkedList<ContentCallback>,
+  low: LinkedList<ContentCallback>,
+];
+
+/**
+ * A priority-based queue for storage content streaming.
+ *
+ * This queue manages content streaming for all priority levels (HIGH, MEDIUM, LOW).
+ * Content is processed in priority order: HIGH first, then MEDIUM, then LOW.
+ *
+ * Key features:
+ * - Stores callbacks to get content (lazy evaluation) rather than content itself
+ * - Priority-based ordering: HIGH > MEDIUM > LOW
+ */
+export class StorageStreamingQueue {
+  private queues: StreamingQueueTuple;
+
+  constructor() {
+    this.queues = [
+      meteredList("storage-streaming", { priority: CO_VALUE_PRIORITY.HIGH }),
+      meteredList("storage-streaming", { priority: CO_VALUE_PRIORITY.MEDIUM }),
+      meteredList("storage-streaming", { priority: CO_VALUE_PRIORITY.LOW }),
+    ];
+  }
+
+  private getQueue(priority: CoValuePriority) {
+    return this.queues[PRIORITY_TO_QUEUE_INDEX[priority]];
+  }
+
+  /**
+   * Push a content callback to the queue with explicit priority.
+   * The callback will be invoked when the entry is pulled and processed.
+   *
+   * @param entry - Callback that pushes content when invoked
+   * @param priority - Priority for this entry (HIGH, MEDIUM, or LOW)
+   */
+  public push(entry: ContentCallback, priority: CoValuePriority): void {
+    this.getQueue(priority).push(entry);
+  }
+
+  /**
+   * Pull the next entry from the queue.
+   * Returns undefined if no entries are available.
+   * Priority order: HIGH > MEDIUM > LOW
+   */
+  public pull(): ContentCallback | undefined {
+    // Find the first non-empty queue (HIGH > MEDIUM > LOW)
+    const queueIndex = this.queues.findIndex((queue) => queue.length > 0);
+
+    if (queueIndex === -1) {
+      return undefined;
+    }
+
+    const entry = this.queues[queueIndex]?.shift();
+
+    if (!entry) {
+      return undefined;
+    }
+
+    return entry;
+  }
+
+  /**
+   * Check if the queue is empty (no pending entries).
+   */
+  public isEmpty(): boolean {
+    return this.queues.every((queue) => queue.length === 0);
+  }
+
+  private listener: (() => void) | undefined;
+  setListener(listener: () => void): void {
+    this.listener = listener;
+  }
+
+  emit(): void {
+    this.listener?.();
+  }
+}

package/src/storage/knownState.ts

@@ -11,14 +11,14 @@ import {
  * and provides the API to wait for the data to be fully stored.
  */
 export class StorageKnownState {
-  knwonStates = new Map<string, CoValueKnownState>();
+  knownStates = new Map<string, CoValueKnownState>();
 
   getKnownState(id: string): CoValueKnownState {
-    const knownState = this.knwonStates.get(id);
+    const knownState = this.knownStates.get(id);
 
     if (!knownState) {
       const empty = emptyKnownState(id as RawCoID);
-      this.knwonStates.set(id, empty);
+      this.knownStates.set(id, empty);
       return empty;
     }
 
@@ -26,7 +26,7 @@ export class StorageKnownState {
   }
 
   setKnownState(id: string, knownState: CoValueKnownState) {
-    this.knwonStates.set(id, knownState);
+    this.knownStates.set(id, knownState);
   }
 
   handleUpdate(id: string, knownState: CoValueKnownState) {

package/src/storage/sqlite/client.ts

@@ -4,6 +4,7 @@ import type {
 } from "../../coValueCore/verifiedState.js";
 import type { Signature } from "../../crypto/crypto.js";
 import type { RawCoID, SessionID } from "../../exports.js";
+import type { PeerID } from "../../sync.js";
 import { logger } from "../../logger.js";
 import type {
   DBClientInterfaceSync,
@@ -193,4 +194,34 @@ export class SQLiteClient
     this.db.transaction(() => operationsCallback(this));
     return undefined;
   }
+
+  getUnsyncedCoValueIDs(): RawCoID[] {
+    const rows = this.db.query<{ co_value_id: RawCoID }>(
+      "SELECT DISTINCT co_value_id FROM unsynced_covalues",
+      [],
+    ) as { co_value_id: RawCoID }[];
+    return rows.map((row) => row.co_value_id);
+  }
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): void {
+    for (const update of updates) {
+      if (update.synced) {
+        this.db.run(
+          "DELETE FROM unsynced_covalues WHERE co_value_id = ? AND peer_id = ?",
+          [update.id, update.peerId],
+        );
+      } else {
+        this.db.run(
+          "INSERT OR REPLACE INTO unsynced_covalues (co_value_id, peer_id) VALUES (?, ?)",
+          [update.id, update.peerId],
+        );
+      }
+    }
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [id]);
+  }
 }