cojson 0.19.18 → 0.19.19

package/src/UnsyncedCoValuesTracker.ts

@@ -0,0 +1,272 @@
+import type { RawCoID } from "./ids.js";
+import { logger } from "./logger.js";
+import type { PeerID } from "./sync.js";
+import type { StorageAPI } from "./storage/types.js";
+
+/**
+ * Used to track a CoValue that hasn't been synced to any peer,
+ * because none is currently connected.
+ */
+const ANY_PEER_ID: PeerID = "any";
+
+// Flush pending updates to storage after 200ms
+let BATCH_DELAY_MS = 200;
+
+/**
+ * Set the delay for flushing pending sync state updates to storage.
+ * @internal
+ */
+export function setSyncStateTrackingBatchDelay(delay: number): void {
+  BATCH_DELAY_MS = delay;
+}
+
+type PendingUpdate = {
+  id: RawCoID;
+  peerId: PeerID;
+  synced: boolean;
+};
+
+/**
+ * Tracks CoValues that have unsynced changes to specific peers.
+ * Maintains an in-memory map and periodically persists to storage.
+ */
+export class UnsyncedCoValuesTracker {
+  private unsynced: Map<RawCoID, Set<PeerID>> = new Map();
+  private coValueListeners: Map<RawCoID, Set<(synced: boolean) => void>> =
+    new Map();
+  // Listeners for global "all synced" status changes
+  private globalListeners: Set<(synced: boolean) => void> = new Set();
+
+  // Pending updates to be persisted
+  private pendingUpdates: PendingUpdate[] = [];
+  private flushTimer: ReturnType<typeof setTimeout> | undefined;
+
+  private storage?: StorageAPI;
+
+  /**
+   * Add a CoValue as unsynced to a specific peer.
+   * Triggers persistence if storage is available.
+   * @returns true if the CoValue was already tracked, false otherwise.
+   */
+  add(id: RawCoID, peerId: PeerID = ANY_PEER_ID): boolean {
+    if (!this.unsynced.has(id)) {
+      this.unsynced.set(id, new Set());
+    }
+    const peerSet = this.unsynced.get(id)!;
+
+    const alreadyTracked = peerSet.has(peerId);
+    if (!alreadyTracked) {
+      // Only update if this is a new peer
+      peerSet.add(peerId);
+
+      this.schedulePersist(id, peerId, false);
+
+      this.notifyCoValueListeners(id, false);
+      this.notifyGlobalListeners(false);
+    }
+
+    return alreadyTracked;
+  }
+
+  /**
+   * Remove a CoValue from being unsynced to a specific peer.
+   * Triggers persistence if storage is available.
+   */
+  remove(id: RawCoID, peerId: PeerID = ANY_PEER_ID): void {
+    const peerSet = this.unsynced.get(id);
+    if (!peerSet || !peerSet.has(peerId)) {
+      return;
+    }
+
+    peerSet.delete(peerId);
+
+    // If no more unsynced peers for this CoValue, remove the entry
+    if (peerSet.size === 0) {
+      this.unsynced.delete(id);
+    }
+
+    this.schedulePersist(id, peerId, true);
+
+    const isSynced = !this.unsynced.has(id);
+    this.notifyCoValueListeners(id, isSynced);
+    this.notifyGlobalListeners(this.isAllSynced());
+  }
+
+  /**
+   * Remove all tracking for a CoValue (all peers).
+   * Triggers persistence if storage is available.
+   */
+  removeAll(id: RawCoID): void {
+    const peerSet = this.unsynced.get(id);
+    if (!peerSet) {
+      return;
+    }
+
+    // Remove all peers for this CoValue
+    const peersToRemove = Array.from(peerSet);
+    for (const peerId of peersToRemove) {
+      this.remove(id, peerId);
+    }
+  }
+
+  forcePersist(): Promise<void> | undefined {
+    return this.flush();
+  }
+
+  private schedulePersist(id: RawCoID, peerId: PeerID, synced: boolean): void {
+    const storage = this.storage;
+    if (!storage) {
+      return;
+    }
+
+    this.pendingUpdates.push({ id, peerId, synced });
+    if (!this.flushTimer) {
+      this.flushTimer = setTimeout(() => {
+        this.flush();
+      }, BATCH_DELAY_MS);
+    }
+  }
+
+  /**
+   * Flush all pending persistence updates in a batch
+   */
+  private flush(): Promise<void> | undefined {
+    if (this.flushTimer) {
+      clearTimeout(this.flushTimer);
+      this.flushTimer = undefined;
+    }
+
+    if (this.pendingUpdates.length === 0) {
+      return;
+    }
+
+    const storage = this.storage;
+    if (!storage) {
+      return;
+    }
+
+    const filteredUpdates = this.simplifyPendingUpdates(this.pendingUpdates);
+    this.pendingUpdates = [];
+
+    return new Promise((resolve) => {
+      try {
+        storage.trackCoValuesSyncState(filteredUpdates, () => resolve());
+      } catch (error) {
+        logger.warn("Failed to persist batched unsynced CoValue tracking", {
+          err: error,
+        });
+        resolve();
+      }
+    });
+  }
+
+  /**
+   * Get all CoValue IDs that have at least one unsynced peer.
+   */
+  getAll(): RawCoID[] {
+    return Array.from(this.unsynced.keys());
+  }
+
+  /**
+   * Check if all CoValues are synced
+   */
+  isAllSynced(): boolean {
+    return this.unsynced.size === 0;
+  }
+
+  /**
+   * Check if a specific CoValue is tracked as unsynced.
+   */
+  has(id: RawCoID): boolean {
+    return this.unsynced.has(id);
+  }
+
+  /**
+   * Subscribe to changes in whether a specific CoValue is synced.
+   * The listener is called immediately with the current state.
+   * @returns Unsubscribe function
+   */
+  subscribe(id: RawCoID, listener: (synced: boolean) => void): () => void;
+  /**
+   * Subscribe to changes in whether all CoValues are synced.
+   * The listener is called immediately with the current state.
+   * @returns Unsubscribe function
+   */
+  subscribe(listener: (synced: boolean) => void): () => void;
+  subscribe(
+    idOrListener: RawCoID | ((synced: boolean) => void),
+    listener?: (synced: boolean) => void,
+  ): () => void {
+    if (typeof idOrListener === "string" && listener) {
+      const id = idOrListener;
+      if (!this.coValueListeners.has(id)) {
+        this.coValueListeners.set(id, new Set());
+      }
+      this.coValueListeners.get(id)!.add(listener);
+
+      // Call immediately with current state
+      const isSynced = !this.unsynced.has(id);
+      listener(isSynced);
+
+      return () => {
+        const listeners = this.coValueListeners.get(id);
+        if (listeners) {
+          listeners.delete(listener);
+          if (listeners.size === 0) {
+            this.coValueListeners.delete(id);
+          }
+        }
+      };
+    }
+
+    const globalListener = idOrListener as (synced: boolean) => void;
+    this.globalListeners.add(globalListener);
+
+    // Call immediately with current state
+    globalListener(this.isAllSynced());
+
+    return () => {
+      this.globalListeners.delete(globalListener);
+    };
+  }
+
+  setStorage(storage: StorageAPI) {
+    this.storage = storage;
+  }
+
+  removeStorage() {
+    this.storage = undefined;
+  }
+
+  /**
+   * Notify all listeners for a specific CoValue about sync status change.
+   */
+  private notifyCoValueListeners(id: RawCoID, synced: boolean): void {
+    const listeners = this.coValueListeners.get(id);
+    if (listeners) {
+      for (const listener of listeners) {
+        listener(synced);
+      }
+    }
+  }
+
+  /**
+   * Notify all global listeners about "all synced" status change.
+   */
+  private notifyGlobalListeners(allSynced: boolean): void {
+    for (const listener of this.globalListeners) {
+      listener(allSynced);
+    }
+  }
+
+  /**
+   * Keep only the last update for each (id, peerId) combination
+   */
+  private simplifyPendingUpdates(updates: PendingUpdate[]): PendingUpdate[] {
+    const latestUpdates = new Map<string, PendingUpdate>();
+    for (const update of updates) {
+      latestUpdates.set(`${update.id}|${update.peerId}`, update);
+    }
+    return Array.from(latestUpdates.values());
+  }
+}

package/src/exports.ts

@@ -65,12 +65,13 @@ import type { AgentID, RawCoID, SessionID } from "./ids.js";
 import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { isAccountRole } from "./permissions.js";
-import type { Peer, SyncMessage } from "./sync.js";
+import type { Peer, SyncMessage, SyncWhen } from "./sync.js";
 import {
   DisconnectedError,
   SyncManager,
   hwrServerPeerSelector,
 } from "./sync.js";
+import { setSyncStateTrackingBatchDelay } from "./UnsyncedCoValuesTracker.js";
 import { emptyKnownState } from "./knownState.js";
 
 import {
@@ -125,6 +126,7 @@ export const cojsonInternals = {
   setCoValueLoadingRetryDelay,
   setCoValueLoadingMaxRetries,
   setCoValueLoadingTimeout,
+  setSyncStateTrackingBatchDelay,
   ConnectedPeerChannel,
   textEncoder,
   textDecoder,
@@ -193,6 +195,7 @@ export type {
   AccountRole,
   AvailableCoValueCore,
   PeerState,
+  SyncWhen,
   CoValueHeader,
 };
 

package/src/localNode.ts

@@ -34,7 +34,7 @@ import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID, isRawCoID } from "./ids.js";
 import { logger } from "./logger.js";
 import { StorageAPI } from "./storage/index.js";
-import { Peer, PeerID, SyncManager } from "./sync.js";
+import { Peer, PeerID, SyncManager, type SyncWhen } from "./sync.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
@@ -75,6 +75,7 @@ export class LocalNode {
     agentSecret: AgentSecret,
     currentSessionID: SessionID,
     crypto: CryptoProvider,
+    public readonly syncWhen?: SyncWhen,
   ) {
     this.agentSecret = agentSecret;
     this.currentSessionID = currentSessionID;
@@ -94,11 +95,13 @@ export class LocalNode {
 
   setStorage(storage: StorageAPI) {
     this.storage = storage;
+    this.syncManager.setStorage(storage);
   }
 
   removeStorage() {
     this.storage?.close();
     this.storage = undefined;
+    this.syncManager.removeStorage();
   }
 
   hasCoValue(id: RawCoID) {
@@ -178,12 +181,14 @@ export class LocalNode {
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
     peers?: Peer[];
+    syncWhen?: SyncWhen;
     storage?: StorageAPI;
   }): RawAccount {
     const {
       crypto,
       initialAgentSecret = crypto.newRandomAgentSecret(),
       peers = [],
+      syncWhen,
     } = opts;
     const accountHeader = accountHeaderForInitialAgentSecret(
       initialAgentSecret,
@@ -195,6 +200,7 @@ export class LocalNode {
       initialAgentSecret,
       crypto.newRandomSessionID(accountID as RawAccountID),
       crypto,
+      syncWhen,
     );
 
     if (opts.storage) {
@@ -236,6 +242,7 @@ export class LocalNode {
   static async withNewlyCreatedAccount({
     creationProps,
     peers,
+    syncWhen,
     migration,
     crypto,
     initialAgentSecret = crypto.newRandomAgentSecret(),
@@ -243,6 +250,7 @@ export class LocalNode {
   }: {
     creationProps: { name: string };
     peers?: Peer[];
+    syncWhen?: SyncWhen;
     migration?: RawAccountMigration<AccountMeta>;
     crypto: CryptoProvider;
     initialAgentSecret?: AgentSecret;
@@ -257,6 +265,7 @@ export class LocalNode {
       crypto,
       initialAgentSecret,
       peers,
+      syncWhen,
       storage,
     });
     const node = account.core.node;
@@ -299,6 +308,7 @@ export class LocalNode {
     accountSecret,
     sessionID,
     peers,
+    syncWhen,
     crypto,
     migration,
     storage,
@@ -307,6 +317,7 @@ export class LocalNode {
     accountSecret: AgentSecret;
     sessionID: SessionID | undefined;
     peers: Peer[];
+    syncWhen?: SyncWhen;
     crypto: CryptoProvider;
     migration?: RawAccountMigration<AccountMeta>;
     storage?: StorageAPI;
@@ -318,6 +329,7 @@ export class LocalNode {
         accountSecret,
         sessionID || crypto.newRandomSessionID(accountID),
         crypto,
+        syncWhen,
       );
 
       if (storage) {
@@ -817,9 +829,9 @@ export class LocalNode {
    *
    * @returns Promise of the current pending store operation, if any.
    */
-  gracefulShutdown(): Promise<unknown> | undefined {
-    this.syncManager.gracefulShutdown();
+  async gracefulShutdown(): Promise<unknown> {
     this.garbageCollector?.stop();
+    await this.syncManager.gracefulShutdown();
     return this.storage?.close();
   }
 }

package/src/storage/knownState.ts

@@ -11,14 +11,14 @@ import {
  * and provides the API to wait for the data to be fully stored.
  */
 export class StorageKnownState {
-  knwonStates = new Map<string, CoValueKnownState>();
+  knownStates = new Map<string, CoValueKnownState>();
 
   getKnownState(id: string): CoValueKnownState {
-    const knownState = this.knwonStates.get(id);
+    const knownState = this.knownStates.get(id);
 
     if (!knownState) {
       const empty = emptyKnownState(id as RawCoID);
-      this.knwonStates.set(id, empty);
+      this.knownStates.set(id, empty);
       return empty;
     }
 
@@ -26,7 +26,7 @@ export class StorageKnownState {
   }
 
   setKnownState(id: string, knownState: CoValueKnownState) {
-    this.knwonStates.set(id, knownState);
+    this.knownStates.set(id, knownState);
   }
 
   handleUpdate(id: string, knownState: CoValueKnownState) {

package/src/storage/sqlite/client.ts

@@ -4,6 +4,7 @@ import type {
 } from "../../coValueCore/verifiedState.js";
 import type { Signature } from "../../crypto/crypto.js";
 import type { RawCoID, SessionID } from "../../exports.js";
+import type { PeerID } from "../../sync.js";
 import { logger } from "../../logger.js";
 import type {
   DBClientInterfaceSync,
@@ -193,4 +194,34 @@ export class SQLiteClient
     this.db.transaction(() => operationsCallback(this));
     return undefined;
   }
+
+  getUnsyncedCoValueIDs(): RawCoID[] {
+    const rows = this.db.query<{ co_value_id: RawCoID }>(
+      "SELECT DISTINCT co_value_id FROM unsynced_covalues",
+      [],
+    ) as { co_value_id: RawCoID }[];
+    return rows.map((row) => row.co_value_id);
+  }
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): void {
+    for (const update of updates) {
+      if (update.synced) {
+        this.db.run(
+          "DELETE FROM unsynced_covalues WHERE co_value_id = ? AND peer_id = ?",
+          [update.id, update.peerId],
+        );
+      } else {
+        this.db.run(
+          "INSERT OR REPLACE INTO unsynced_covalues (co_value_id, peer_id) VALUES (?, ?)",
+          [update.id, update.peerId],
+        );
+      }
+    }
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [id]);
+  }
 }

package/src/storage/sqlite/sqliteMigrations.ts

@@ -31,6 +31,15 @@ export const migrations: Record<number, string[]> = {
     ) WITHOUT ROWID;`,
     "ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;",
   ],
+  4: [
+    `CREATE TABLE IF NOT EXISTS unsynced_covalues (
+      rowID INTEGER PRIMARY KEY,
+      co_value_id TEXT NOT NULL,
+      peer_id TEXT NOT NULL,
+      UNIQUE (co_value_id, peer_id)
+    );`,
+    "CREATE INDEX IF NOT EXISTS idx_unsynced_covalues_co_value_id ON unsynced_covalues(co_value_id);",
+  ],
 };
 
 type Migration = {

package/src/storage/sqliteAsync/client.ts

@@ -15,6 +15,7 @@ import type {
   TransactionRow,
 } from "../types.js";
 import type { SQLiteDatabaseDriverAsync } from "./types.js";
+import type { PeerID } from "../../sync.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -201,4 +202,38 @@ export class SQLiteClientAsync
   ) {
     return this.db.transaction(() => operationsCallback(this));
   }
+
+  async getUnsyncedCoValueIDs(): Promise<RawCoID[]> {
+    const rows = await this.db.query<{ co_value_id: RawCoID }>(
+      "SELECT DISTINCT co_value_id FROM unsynced_covalues",
+      [],
+    );
+    return rows.map((row) => row.co_value_id);
+  }
+
+  async trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): Promise<void> {
+    await Promise.all(
+      updates.map(async (update) => {
+        if (update.synced) {
+          await this.db.run(
+            "DELETE FROM unsynced_covalues WHERE co_value_id = ? AND peer_id = ?",
+            [update.id, update.peerId],
+          );
+        } else {
+          await this.db.run(
+            "INSERT OR REPLACE INTO unsynced_covalues (co_value_id, peer_id) VALUES (?, ?)",
+            [update.id, update.peerId],
+          );
+        }
+      }),
+    );
+  }
+
+  async stopTrackingSyncState(id: RawCoID): Promise<void> {
+    await this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [
+      id,
+    ]);
+  }
 }

package/src/storage/storageAsync.ts

@@ -10,7 +10,7 @@ import {
   logger,
 } from "../exports.js";
 import { StoreQueue } from "../queue/StoreQueue.js";
-import { NewContentMessage } from "../sync.js";
+import { NewContentMessage, type PeerID } from "../sync.js";
 import {
   CoValueKnownState,
   emptyKnownState,
@@ -392,6 +392,23 @@ export class StorageApiAsync implements StorageAPI {
     return this.knownStates.waitForSync(id, coValue);
   }
 
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void {
+    this.dbClient.trackCoValuesSyncState(updates).then(() => done?.());
+  }
+
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void {
+    this.dbClient.getUnsyncedCoValueIDs().then(callback);
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.dbClient.stopTrackingSyncState(id);
+  }
+
   close() {
     return this.storeQueue.close();
   }

package/src/storage/storageSync.ts

@@ -10,7 +10,7 @@ import {
   type StorageAPI,
   logger,
 } from "../exports.js";
-import { NewContentMessage } from "../sync.js";
+import { NewContentMessage, type PeerID } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
 import {
   CoValueKnownState,
@@ -365,6 +365,25 @@ export class StorageApiSync implements StorageAPI {
     return this.knownStates.waitForSync(id, coValue);
   }
 
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void {
+    this.dbClient.trackCoValuesSyncState(updates);
+    done?.();
+  }
+
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void {
+    const ids = this.dbClient.getUnsyncedCoValueIDs();
+    callback(ids);
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.dbClient.stopTrackingSyncState(id);
+  }
+
   close() {
     return undefined;
   }

package/src/storage/types.ts

@@ -5,6 +5,7 @@ import type {
 import { Signature } from "../crypto/crypto.js";
 import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
 import { NewContentMessage } from "../sync.js";
+import type { PeerID } from "../sync.js";
 import { CoValueKnownState } from "../knownState.js";
 
 export type CorrectionCallback = (
@@ -29,6 +30,28 @@ export interface StorageAPI {
 
   waitForSync(id: string, coValue: CoValueCore): Promise<void>;
 
+  /**
+   * Track multiple sync status updates.
+   * Does not guarantee the updates will be applied in order, so only one
+   * update per CoValue ID + Peer ID combination should be tracked at a time.
+   */
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void;
+
+  /**
+   * Get all CoValue IDs that have at least one unsynced peer.
+   */
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void;
+
+  /**
+   * Stop tracking sync status for a CoValue (remove all peer entries).
+   */
+  stopTrackingSyncState(id: RawCoID): void;
+
   close(): Promise<unknown> | undefined;
 }
 
@@ -118,6 +141,14 @@ export interface DBClientInterfaceAsync {
   transaction(
     callback: (tx: DBTransactionInterfaceAsync) => Promise<unknown>,
   ): Promise<unknown>;
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): Promise<void>;
+
+  getUnsyncedCoValueIDs(): Promise<RawCoID[]>;
+
+  stopTrackingSyncState(id: RawCoID): Promise<void>;
 }
 
 export interface DBTransactionInterfaceSync {
@@ -170,4 +201,12 @@ export interface DBClientInterfaceSync {
   ): Pick<SignatureAfterRow, "idx" | "signature">[];
 
   transaction(callback: (tx: DBTransactionInterfaceSync) => unknown): unknown;
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): void;
+
+  getUnsyncedCoValueIDs(): RawCoID[];
+
+  stopTrackingSyncState(id: RawCoID): void;
 }