cojson 0.19.19 → 0.19.21

package/src/storage/storageAsync.ts

@@ -36,6 +36,12 @@ export class StorageApiAsync implements StorageAPI {
 
   private loadedCoValues = new Set<RawCoID>();
 
+  // Track pending loads to deduplicate concurrent requests
+  private pendingKnownStateLoads = new Map<
+    string,
+    Promise<CoValueKnownState | undefined>
+  >();
+
   constructor(dbClient: DBClientInterfaceAsync) {
     this.dbClient = dbClient;
   }
@@ -46,6 +52,51 @@ export class StorageApiAsync implements StorageAPI {
     return this.knownStates.getKnownState(id);
   }
 
+  loadKnownState(
+    id: string,
+    callback: (knownState: CoValueKnownState | undefined) => void,
+  ): void {
+    // Check in-memory cache first
+    const cached = this.knownStates.getCachedKnownState(id);
+    if (cached) {
+      callback(cached);
+      return;
+    }
+
+    // Check if there's already a pending load for this ID (deduplication)
+    const pending = this.pendingKnownStateLoads.get(id);
+    if (pending) {
+      // Ensure callback is always called, even if pending fails unexpectedly
+      pending.then(callback, () => callback(undefined));
+      return;
+    }
+
+    // Start new load and track it for deduplication
+    const loadPromise = this.dbClient
+      .getCoValueKnownState(id)
+      .then((knownState) => {
+        if (knownState) {
+          // Cache for future use
+          this.knownStates.setKnownState(id, knownState);
+        }
+        return knownState;
+      })
+      .catch((err) => {
+        // Error handling contract:
+        // - Log warning
+        // - Behave like "not found" so callers can fall back (full load / load from peers)
+        logger.warn("Failed to load knownState from storage", { id, err });
+        return undefined;
+      })
+      .finally(() => {
+        // Remove from pending map after completion (success or failure)
+        this.pendingKnownStateLoads.delete(id);
+      });
+
+    this.pendingKnownStateLoads.set(id, loadPromise);
+    loadPromise.then(callback);
+  }
+
   async load(
     id: string,
     callback: (data: NewContentMessage) => void,

package/src/storage/storageSync.ts

@@ -1,4 +1,3 @@
-import { UpDownCounter, metrics } from "@opentelemetry/api";
 import {
   createContentMessage,
   exceedsRecommendedSize,
@@ -30,22 +29,25 @@ import type {
   StoredCoValueRow,
   StoredSessionRow,
 } from "./types.js";
+import {
+  ContentCallback,
+  StorageStreamingQueue,
+} from "../queue/StorageStreamingQueue.js";
+import { getPriorityFromHeader } from "../priority.js";
 
 export class StorageApiSync implements StorageAPI {
-  private streamingCounter: UpDownCounter;
-
   private readonly dbClient: DBClientInterfaceSync;
   private loadedCoValues = new Set<RawCoID>();
 
+  /**
+   * Queue for streaming content that will be pulled by SyncManager.
+   * Only used when content requires streaming (multiple chunks).
+   */
+  readonly streamingQueue: StorageStreamingQueue;
+
   constructor(dbClient: DBClientInterfaceSync) {
     this.dbClient = dbClient;
-    this.streamingCounter = metrics
-      .getMeter("cojson")
-      .createUpDownCounter(`jazz.storage.streaming`, {
-        description: "Number of streaming coValues",
-        unit: "1",
-      });
-    this.streamingCounter.add(0);
+    this.streamingQueue = new StorageStreamingQueue();
   }
 
   knownStates = new StorageKnownState();
@@ -54,6 +56,28 @@ export class StorageApiSync implements StorageAPI {
     return this.knownStates.getKnownState(id);
   }
 
+  loadKnownState(
+    id: string,
+    callback: (knownState: CoValueKnownState | undefined) => void,
+  ): void {
+    // Check in-memory cache first
+    const cached = this.knownStates.getCachedKnownState(id);
+    if (cached) {
+      callback(cached);
+      return;
+    }
+
+    // Load from database
+    const knownState = this.dbClient.getCoValueKnownState(id);
+
+    if (knownState) {
+      // Cache for future use
+      this.knownStates.setKnownState(id, knownState);
+    }
+
+    callback(knownState);
+  }
+
   async load(
     id: string,
     callback: (data: NewContentMessage) => void,
@@ -62,7 +86,7 @@ export class StorageApiSync implements StorageAPI {
     await this.loadCoValue(id, callback, done);
   }
 
-  async loadCoValue(
+  loadCoValue(
     id: string,
     callback: (data: NewContentMessage) => void,
     done?: (found: boolean) => void,
@@ -89,8 +113,18 @@ export class StorageApiSync implements StorageAPI {
 
       if (signatures.length > 0) {
         contentStreaming = true;
-        signaturesBySession.set(sessionRow.sessionID, signatures);
       }
+
+      const lastSignature = signatures[signatures.length - 1];
+
+      if (lastSignature?.signature !== sessionRow.lastSignature) {
+        signatures.push({
+          idx: sessionRow.lastIdx,
+          signature: sessionRow.lastSignature,
+        });
+      }
+
+      signaturesBySession.set(sessionRow.sessionID, signatures);
     }
 
     const knownState = this.knownStates.getKnownState(coValueRow.id);
@@ -106,78 +140,110 @@ export class StorageApiSync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
+    const priority = getPriorityFromHeader(coValueRow.header);
+    const contentMessage = createContentMessage(
+      coValueRow.id,
+      coValueRow.header,
+    );
 
     if (contentStreaming) {
-      this.streamingCounter.add(1);
       contentMessage.expectContentUntil = knownState.sessions;
     }
 
+    const streamingQueue: ContentCallback[] = [];
+
     for (const sessionRow of allCoValueSessions) {
-      const signatures = signaturesBySession.get(sessionRow.sessionID) || [];
+      const signatures = signaturesBySession.get(sessionRow.sessionID);
 
-      let idx = 0;
+      if (!signatures) {
+        throw new Error("Signatures not found for session");
+      }
 
-      const lastSignature = signatures[signatures.length - 1];
+      const firstSignature = signatures[0];
 
-      if (lastSignature?.signature !== sessionRow.lastSignature) {
-        signatures.push({
-          idx: sessionRow.lastIdx,
-          signature: sessionRow.lastSignature,
-        });
+      if (!firstSignature) {
+        continue;
       }
 
-      for (const signature of signatures) {
-        const newTxsInSession = this.dbClient.getNewTransactionInSession(
-          sessionRow.rowID,
-          idx,
-          signature.idx,
-        );
+      this.loadSessionTransactions(
+        contentMessage,
+        sessionRow,
+        0,
+        firstSignature,
+      );
 
-        collectNewTxs({
-          newTxsInSession,
-          contentMessage,
-          sessionRow,
-          firstNewTxIdx: idx,
-          signature: signature.signature,
-        });
+      for (let i = 1; i < signatures.length; i++) {
+        const prevSignature = signatures[i - 1];
 
-        idx = signature.idx + 1;
+        if (!prevSignature) {
+          throw new Error("Previous signature is nullish");
+        }
 
-        if (signatures.length > 1) {
-          this.pushContentWithDependencies(
-            coValueRow,
-            contentMessage,
-            callback,
-          );
-          contentMessage = createContentMessage(
+        streamingQueue.push(() => {
+          const contentMessage = createContentMessage(
             coValueRow.id,
             coValueRow.header,
           );
 
-          // Introduce a delay to not block the main thread
-          // for the entire content processing
-          await new Promise((resolve) => setTimeout(resolve));
-        }
+          const signature = signatures[i];
+          if (!signature) throw new Error("Signature item is nullish");
+
+          this.loadSessionTransactions(
+            contentMessage,
+            sessionRow,
+            prevSignature.idx + 1,
+            signature,
+          );
+
+          if (Object.keys(contentMessage.new).length > 0) {
+            this.pushContentWithDependencies(
+              coValueRow,
+              contentMessage,
+              callback,
+            );
+          }
+        });
       }
     }
 
-    const hasNewContent = Object.keys(contentMessage.new).length > 0;
+    // Send the first chunk
+    this.pushContentWithDependencies(coValueRow, contentMessage, callback);
+    this.knownStates.handleUpdate(coValueRow.id, knownState);
 
-    // If there is no new content but steaming is not active, it's the case for a coValue with the header but no transactions
-    // For streaming the push has already been done in the loop above
-    if (hasNewContent || !contentStreaming) {
-      this.pushContentWithDependencies(coValueRow, contentMessage, callback);
+    // All priorities go through the queue (HIGH > MEDIUM > LOW)
+    for (const pushStreamingContent of streamingQueue) {
+      this.streamingQueue.push(pushStreamingContent, priority);
     }
 
-    if (contentStreaming) {
-      this.streamingCounter.add(-1);
+    // Trigger the queue to process the entries
+    if (streamingQueue.length > 0) {
+      this.streamingQueue.emit();
     }
 
-    this.knownStates.handleUpdate(coValueRow.id, knownState);
     done?.(true);
   }
 
+  private loadSessionTransactions(
+    contentMessage: NewContentMessage,
+    sessionRow: StoredSessionRow,
+    idx: number,
+    signature: Pick<SignatureAfterRow, "idx" | "signature">,
+  ) {
+    const newTxsInSession = this.dbClient.getNewTransactionInSession(
+      sessionRow.rowID,
+      idx,
+      signature.idx,
+    );
+
+    collectNewTxs({
+      newTxsInSession,
+      contentMessage,
+      sessionRow,
+      firstNewTxIdx: idx,
+      signature: signature.signature,
+    });
+  }
+
   async pushContentWithDependencies(
     coValueRow: StoredCoValueRow,
     contentMessage: NewContentMessage,

package/src/storage/types.ts

@@ -7,6 +7,7 @@ import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
 import { NewContentMessage } from "../sync.js";
 import type { PeerID } from "../sync.js";
 import { CoValueKnownState } from "../knownState.js";
+import { StorageStreamingQueue } from "../queue/StorageStreamingQueue.js";
 
 export type CorrectionCallback = (
   correction: CoValueKnownState,
@@ -26,6 +27,8 @@ export interface StorageAPI {
   ): void;
   store(data: NewContentMessage, handleCorrection: CorrectionCallback): void;
 
+  streamingQueue?: StorageStreamingQueue;
+
   getKnownState(id: string): CoValueKnownState;
 
   waitForSync(id: string, coValue: CoValueCore): Promise<void>;
@@ -52,6 +55,18 @@ export interface StorageAPI {
    */
   stopTrackingSyncState(id: RawCoID): void;
 
+  /**
+   * Load only the knownState (header presence + session counters) for a CoValue.
+   * This is more efficient than load() when we only need to check if a peer needs new content.
+   *
+   * @param id - The CoValue ID
+   * @param callback - Called with the knownState, or undefined if CoValue not found
+   */
+  loadKnownState(
+    id: string,
+    callback: (knownState: CoValueKnownState | undefined) => void,
+  ): void;
+
   close(): Promise<unknown> | undefined;
 }
 
@@ -149,6 +164,14 @@ export interface DBClientInterfaceAsync {
   getUnsyncedCoValueIDs(): Promise<RawCoID[]>;
 
   stopTrackingSyncState(id: RawCoID): Promise<void>;
+
+  /**
+   * Get the knownState for a CoValue without loading transactions.
+   * Returns undefined if the CoValue doesn't exist.
+   */
+  getCoValueKnownState(
+    coValueId: string,
+  ): Promise<CoValueKnownState | undefined>;
 }
 
 export interface DBTransactionInterfaceSync {
@@ -209,4 +232,10 @@ export interface DBClientInterfaceSync {
   getUnsyncedCoValueIDs(): RawCoID[];
 
   stopTrackingSyncState(id: RawCoID): void;
+
+  /**
+   * Get the knownState for a CoValue without loading transactions.
+   * Returns undefined if the CoValue doesn't exist.
+   */
+  getCoValueKnownState(coValueId: string): CoValueKnownState | undefined;
 }