cojson 0.19.18 → 0.19.20

package/src/storage/sqlite/sqliteMigrations.ts

@@ -31,6 +31,15 @@ export const migrations: Record<number, string[]> = {
     ) WITHOUT ROWID;`,
     "ALTER TABLE sessions ADD COLUMN bytesSinceLastSignature INTEGER;",
   ],
+  4: [
+    `CREATE TABLE IF NOT EXISTS unsynced_covalues (
+      rowID INTEGER PRIMARY KEY,
+      co_value_id TEXT NOT NULL,
+      peer_id TEXT NOT NULL,
+      UNIQUE (co_value_id, peer_id)
+    );`,
+    "CREATE INDEX IF NOT EXISTS idx_unsynced_covalues_co_value_id ON unsynced_covalues(co_value_id);",
+  ],
 };
 
 type Migration = {

package/src/storage/sqliteAsync/client.ts

@@ -15,6 +15,7 @@ import type {
   TransactionRow,
 } from "../types.js";
 import type { SQLiteDatabaseDriverAsync } from "./types.js";
+import type { PeerID } from "../../sync.js";
 
 export type RawCoValueRow = {
   id: RawCoID;
@@ -201,4 +202,38 @@ export class SQLiteClientAsync
   ) {
     return this.db.transaction(() => operationsCallback(this));
   }
+
+  async getUnsyncedCoValueIDs(): Promise<RawCoID[]> {
+    const rows = await this.db.query<{ co_value_id: RawCoID }>(
+      "SELECT DISTINCT co_value_id FROM unsynced_covalues",
+      [],
+    );
+    return rows.map((row) => row.co_value_id);
+  }
+
+  async trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): Promise<void> {
+    await Promise.all(
+      updates.map(async (update) => {
+        if (update.synced) {
+          await this.db.run(
+            "DELETE FROM unsynced_covalues WHERE co_value_id = ? AND peer_id = ?",
+            [update.id, update.peerId],
+          );
+        } else {
+          await this.db.run(
+            "INSERT OR REPLACE INTO unsynced_covalues (co_value_id, peer_id) VALUES (?, ?)",
+            [update.id, update.peerId],
+          );
+        }
+      }),
+    );
+  }
+
+  async stopTrackingSyncState(id: RawCoID): Promise<void> {
+    await this.db.run("DELETE FROM unsynced_covalues WHERE co_value_id = ?", [
+      id,
+    ]);
+  }
 }

package/src/storage/storageAsync.ts

@@ -10,7 +10,7 @@ import {
   logger,
 } from "../exports.js";
 import { StoreQueue } from "../queue/StoreQueue.js";
-import { NewContentMessage } from "../sync.js";
+import { NewContentMessage, type PeerID } from "../sync.js";
 import {
   CoValueKnownState,
   emptyKnownState,
@@ -392,6 +392,23 @@ export class StorageApiAsync implements StorageAPI {
     return this.knownStates.waitForSync(id, coValue);
   }
 
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void {
+    this.dbClient.trackCoValuesSyncState(updates).then(() => done?.());
+  }
+
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void {
+    this.dbClient.getUnsyncedCoValueIDs().then(callback);
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.dbClient.stopTrackingSyncState(id);
+  }
+
   close() {
     return this.storeQueue.close();
   }

package/src/storage/storageSync.ts

@@ -1,4 +1,3 @@
-import { UpDownCounter, metrics } from "@opentelemetry/api";
 import {
   createContentMessage,
   exceedsRecommendedSize,
@@ -10,7 +9,7 @@ import {
   type StorageAPI,
   logger,
 } from "../exports.js";
-import { NewContentMessage } from "../sync.js";
+import { NewContentMessage, type PeerID } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
 import {
   CoValueKnownState,
@@ -30,22 +29,25 @@ import type {
   StoredCoValueRow,
   StoredSessionRow,
 } from "./types.js";
+import {
+  ContentCallback,
+  StorageStreamingQueue,
+} from "../queue/StorageStreamingQueue.js";
+import { getPriorityFromHeader } from "../priority.js";
 
 export class StorageApiSync implements StorageAPI {
-  private streamingCounter: UpDownCounter;
-
   private readonly dbClient: DBClientInterfaceSync;
   private loadedCoValues = new Set<RawCoID>();
 
+  /**
+   * Queue for streaming content that will be pulled by SyncManager.
+   * Only used when content requires streaming (multiple chunks).
+   */
+  readonly streamingQueue: StorageStreamingQueue;
+
   constructor(dbClient: DBClientInterfaceSync) {
     this.dbClient = dbClient;
-    this.streamingCounter = metrics
-      .getMeter("cojson")
-      .createUpDownCounter(`jazz.storage.streaming`, {
-        description: "Number of streaming coValues",
-        unit: "1",
-      });
-    this.streamingCounter.add(0);
+    this.streamingQueue = new StorageStreamingQueue();
   }
 
   knownStates = new StorageKnownState();
@@ -62,7 +64,7 @@ export class StorageApiSync implements StorageAPI {
     await this.loadCoValue(id, callback, done);
   }
 
-  async loadCoValue(
+  loadCoValue(
     id: string,
     callback: (data: NewContentMessage) => void,
     done?: (found: boolean) => void,
@@ -89,8 +91,18 @@ export class StorageApiSync implements StorageAPI {
 
       if (signatures.length > 0) {
         contentStreaming = true;
-        signaturesBySession.set(sessionRow.sessionID, signatures);
       }
+
+      const lastSignature = signatures[signatures.length - 1];
+
+      if (lastSignature?.signature !== sessionRow.lastSignature) {
+        signatures.push({
+          idx: sessionRow.lastIdx,
+          signature: sessionRow.lastSignature,
+        });
+      }
+
+      signaturesBySession.set(sessionRow.sessionID, signatures);
     }
 
     const knownState = this.knownStates.getKnownState(coValueRow.id);
@@ -106,78 +118,110 @@ export class StorageApiSync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
+    const priority = getPriorityFromHeader(coValueRow.header);
+    const contentMessage = createContentMessage(
+      coValueRow.id,
+      coValueRow.header,
+    );
 
     if (contentStreaming) {
-      this.streamingCounter.add(1);
       contentMessage.expectContentUntil = knownState.sessions;
     }
 
+    const streamingQueue: ContentCallback[] = [];
+
     for (const sessionRow of allCoValueSessions) {
-      const signatures = signaturesBySession.get(sessionRow.sessionID) || [];
+      const signatures = signaturesBySession.get(sessionRow.sessionID);
 
-      let idx = 0;
+      if (!signatures) {
+        throw new Error("Signatures not found for session");
+      }
 
-      const lastSignature = signatures[signatures.length - 1];
+      const firstSignature = signatures[0];
 
-      if (lastSignature?.signature !== sessionRow.lastSignature) {
-        signatures.push({
-          idx: sessionRow.lastIdx,
-          signature: sessionRow.lastSignature,
-        });
+      if (!firstSignature) {
+        continue;
       }
 
-      for (const signature of signatures) {
-        const newTxsInSession = this.dbClient.getNewTransactionInSession(
-          sessionRow.rowID,
-          idx,
-          signature.idx,
-        );
+      this.loadSessionTransactions(
+        contentMessage,
+        sessionRow,
+        0,
+        firstSignature,
+      );
 
-        collectNewTxs({
-          newTxsInSession,
-          contentMessage,
-          sessionRow,
-          firstNewTxIdx: idx,
-          signature: signature.signature,
-        });
+      for (let i = 1; i < signatures.length; i++) {
+        const prevSignature = signatures[i - 1];
 
-        idx = signature.idx + 1;
+        if (!prevSignature) {
+          throw new Error("Previous signature is nullish");
+        }
 
-        if (signatures.length > 1) {
-          this.pushContentWithDependencies(
-            coValueRow,
-            contentMessage,
-            callback,
-          );
-          contentMessage = createContentMessage(
+        streamingQueue.push(() => {
+          const contentMessage = createContentMessage(
             coValueRow.id,
             coValueRow.header,
           );
 
-          // Introduce a delay to not block the main thread
-          // for the entire content processing
-          await new Promise((resolve) => setTimeout(resolve));
-        }
+          const signature = signatures[i];
+          if (!signature) throw new Error("Signature item is nullish");
+
+          this.loadSessionTransactions(
+            contentMessage,
+            sessionRow,
+            prevSignature.idx + 1,
+            signature,
+          );
+
+          if (Object.keys(contentMessage.new).length > 0) {
+            this.pushContentWithDependencies(
+              coValueRow,
+              contentMessage,
+              callback,
+            );
+          }
+        });
       }
     }
 
-    const hasNewContent = Object.keys(contentMessage.new).length > 0;
+    // Send the first chunk
+    this.pushContentWithDependencies(coValueRow, contentMessage, callback);
+    this.knownStates.handleUpdate(coValueRow.id, knownState);
 
-    // If there is no new content but steaming is not active, it's the case for a coValue with the header but no transactions
-    // For streaming the push has already been done in the loop above
-    if (hasNewContent || !contentStreaming) {
-      this.pushContentWithDependencies(coValueRow, contentMessage, callback);
+    // All priorities go through the queue (HIGH > MEDIUM > LOW)
+    for (const pushStreamingContent of streamingQueue) {
+      this.streamingQueue.push(pushStreamingContent, priority);
     }
 
-    if (contentStreaming) {
-      this.streamingCounter.add(-1);
+    // Trigger the queue to process the entries
+    if (streamingQueue.length > 0) {
+      this.streamingQueue.emit();
     }
 
-    this.knownStates.handleUpdate(coValueRow.id, knownState);
     done?.(true);
   }
 
+  private loadSessionTransactions(
+    contentMessage: NewContentMessage,
+    sessionRow: StoredSessionRow,
+    idx: number,
+    signature: Pick<SignatureAfterRow, "idx" | "signature">,
+  ) {
+    const newTxsInSession = this.dbClient.getNewTransactionInSession(
+      sessionRow.rowID,
+      idx,
+      signature.idx,
+    );
+
+    collectNewTxs({
+      newTxsInSession,
+      contentMessage,
+      sessionRow,
+      firstNewTxIdx: idx,
+      signature: signature.signature,
+    });
+  }
+
   async pushContentWithDependencies(
     coValueRow: StoredCoValueRow,
     contentMessage: NewContentMessage,
@@ -365,6 +409,25 @@ export class StorageApiSync implements StorageAPI {
     return this.knownStates.waitForSync(id, coValue);
   }
 
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void {
+    this.dbClient.trackCoValuesSyncState(updates);
+    done?.();
+  }
+
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void {
+    const ids = this.dbClient.getUnsyncedCoValueIDs();
+    callback(ids);
+  }
+
+  stopTrackingSyncState(id: RawCoID): void {
+    this.dbClient.stopTrackingSyncState(id);
+  }
+
   close() {
     return undefined;
   }

package/src/storage/types.ts

@@ -5,7 +5,9 @@ import type {
 import { Signature } from "../crypto/crypto.js";
 import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
 import { NewContentMessage } from "../sync.js";
+import type { PeerID } from "../sync.js";
 import { CoValueKnownState } from "../knownState.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
 
 export type CorrectionCallback = (
   correction: CoValueKnownState,
@@ -25,10 +27,34 @@ export interface StorageAPI {
   ): void;
   store(data: NewContentMessage, handleCorrection: CorrectionCallback): void;
 
+  streamingQueue?: StorageStreamingQueue;
+
   getKnownState(id: string): CoValueKnownState;
 
   waitForSync(id: string, coValue: CoValueCore): Promise<void>;
 
+  /**
+   * Track multiple sync status updates.
+   * Does not guarantee the updates will be applied in order, so only one
+   * update per CoValue ID + Peer ID combination should be tracked at a time.
+   */
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+    done?: () => void,
+  ): void;
+
+  /**
+   * Get all CoValue IDs that have at least one unsynced peer.
+   */
+  getUnsyncedCoValueIDs(
+    callback: (unsyncedCoValueIDs: RawCoID[]) => void,
+  ): void;
+
+  /**
+   * Stop tracking sync status for a CoValue (remove all peer entries).
+   */
+  stopTrackingSyncState(id: RawCoID): void;
+
   close(): Promise<unknown> | undefined;
 }
 
@@ -118,6 +144,14 @@ export interface DBClientInterfaceAsync {
   transaction(
     callback: (tx: DBTransactionInterfaceAsync) => Promise<unknown>,
   ): Promise<unknown>;
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): Promise<void>;
+
+  getUnsyncedCoValueIDs(): Promise<RawCoID[]>;
+
+  stopTrackingSyncState(id: RawCoID): Promise<void>;
 }
 
 export interface DBTransactionInterfaceSync {
@@ -170,4 +204,12 @@ export interface DBClientInterfaceSync {
   ): Pick<SignatureAfterRow, "idx" | "signature">[];
 
   transaction(callback: (tx: DBTransactionInterfaceSync) => unknown): unknown;
+
+  trackCoValuesSyncState(
+    updates: { id: RawCoID; peerId: PeerID; synced: boolean }[],
+  ): void;
+
+  getUnsyncedCoValueIDs(): RawCoID[];
+
+  stopTrackingSyncState(id: RawCoID): void;
 }