cojson 0.16.3 → 0.16.4

package/src/storage/storageSync.ts

@@ -1,20 +1,29 @@
 import { UpDownCounter, metrics } from "@opentelemetry/api";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
   RawCoID,
   type SessionID,
   type StorageAPI,
+  logger,
 } from "../exports.js";
-import { getPriorityFromHeader } from "../priority.js";
 import {
   CoValueKnownState,
   NewContentMessage,
   emptyKnownState,
 } from "../sync.js";
 import { StorageKnownState } from "./knownState.js";
-import { collectNewTxs, getDependedOnCoValues } from "./syncUtils.js";
+import {
+  collectNewTxs,
+  getDependedOnCoValues,
+  getNewTransactionsSize,
+} from "./syncUtils.js";
 import type {
+  CorrectionCallback,
   DBClientInterfaceSync,
   SignatureAfterRow,
   StoredCoValueRow,
@@ -84,6 +93,7 @@ export class StorageApiSync implements StorageAPI {
     }
 
     const knownState = this.knwonStates.getKnownState(coValueRow.id);
+    knownState.header = true;
 
     for (const sessionRow of allCoValueSessions) {
       knownState.sessions[sessionRow.sessionID] = sessionRow.lastIdx;
@@ -91,13 +101,7 @@ export class StorageApiSync implements StorageAPI {
 
     this.loadedCoValues.add(coValueRow.id);
 
-    let contentMessage = {
-      action: "content",
-      id: coValueRow.id,
-      header: coValueRow.header,
-      new: {},
-      priority: getPriorityFromHeader(coValueRow.header),
-    } as NewContentMessage;
+    let contentMessage = createContentMessage(coValueRow.id, coValueRow.header);
 
     if (contentStreaming) {
       this.streamingCounter.add(1);
@@ -137,13 +141,10 @@ export class StorageApiSync implements StorageAPI {
             contentMessage,
             callback,
           );
-          contentMessage = {
-            action: "content",
-            id: coValueRow.id,
-            header: coValueRow.header,
-            new: {},
-            priority: getPriorityFromHeader(coValueRow.header),
-          } satisfies NewContentMessage;
+          contentMessage = createContentMessage(
+            coValueRow.id,
+            coValueRow.header,
+          );
 
           // Introduce a delay to not block the main thread
           // for the entire content processing
@@ -189,22 +190,49 @@ export class StorageApiSync implements StorageAPI {
     pushCallback(contentMessage);
   }
 
-  store(
-    msgs: NewContentMessage[],
-    correctionCallback: (data: CoValueKnownState) => void,
+  store(msg: NewContentMessage, correctionCallback: CorrectionCallback) {
+    return this.storeSingle(msg, correctionCallback);
+  }
+
+  /**
+   * This function is called when the storage lacks the information required to store the incoming content.
+   *
+   * It triggers a `correctionCallback` to ask the syncManager to provide the missing information.
+   */
+  private handleCorrection(
+    knownState: CoValueKnownState,
+    correctionCallback: CorrectionCallback,
   ) {
-    for (const msg of msgs) {
-      const success = this.storeSingle(msg, correctionCallback);
+    const correction = correctionCallback(knownState);
+
+    if (!correction) {
+      logger.error("Correction callback returned undefined", {
+        knownState,
+        correction: correction ?? null,
+      });
+      return false;
+    }
+
+    for (const msg of correction) {
+      const success = this.storeSingle(msg, (knownState) => {
+        logger.error("Double correction requested", {
+          msg,
+          knownState,
+        });
+        return undefined;
+      });
 
       if (!success) {
         return false;
       }
     }
+
+    return true;
   }
 
   private storeSingle(
     msg: NewContentMessage,
-    correctionCallback: (data: CoValueKnownState) => void,
+    correctionCallback: CorrectionCallback,
   ): boolean {
     const id = msg.id;
     const coValueRow = this.dbClient.getCoValue(id);
@@ -214,11 +242,9 @@ export class StorageApiSync implements StorageAPI {
 
     if (invalidAssumptionOnHeaderPresence) {
       const knownState = emptyKnownState(id as RawCoID);
-      correctionCallback(knownState);
-
       this.knwonStates.setKnownState(id, knownState);
 
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     const storedCoValueRowID: number = coValueRow
@@ -258,8 +284,7 @@ export class StorageApiSync implements StorageAPI {
     this.knwonStates.handleUpdate(id, knownState);
 
     if (invalidAssumptions) {
-      correctionCallback(knownState);
-      return false;
+      return this.handleCorrection(knownState, correctionCallback);
     }
 
     return true;
@@ -272,35 +297,29 @@ export class StorageApiSync implements StorageAPI {
     storedCoValueRowID: number,
   ) {
     const newTransactions = msg.new[sessionID]?.newTransactions || [];
+    const lastIdx = sessionRow?.lastIdx || 0;
 
-    const actuallyNewOffset =
-      (sessionRow?.lastIdx || 0) - (msg.new[sessionID]?.after || 0);
+    const actuallyNewOffset = lastIdx - (msg.new[sessionID]?.after || 0);
 
     const actuallyNewTransactions = newTransactions.slice(actuallyNewOffset);
 
     if (actuallyNewTransactions.length === 0) {
-      return sessionRow?.lastIdx || 0;
+      return lastIdx;
     }
 
-    let newBytesSinceLastSignature =
-      (sessionRow?.bytesSinceLastSignature || 0) +
-      actuallyNewTransactions.reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+    let bytesSinceLastSignature = sessionRow?.bytesSinceLastSignature || 0;
+    const newTransactionsSize = getNewTransactionsSize(actuallyNewTransactions);
 
     const newLastIdx =
       (sessionRow?.lastIdx || 0) + actuallyNewTransactions.length;
 
     let shouldWriteSignature = false;
 
-    if (newBytesSinceLastSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(bytesSinceLastSignature, newTransactionsSize)) {
       shouldWriteSignature = true;
-      newBytesSinceLastSignature = 0;
+      bytesSinceLastSignature = 0;
+    } else {
+      bytesSinceLastSignature += newTransactionsSize;
     }
 
     const nextIdx = sessionRow?.lastIdx || 0;
@@ -312,7 +331,7 @@ export class StorageApiSync implements StorageAPI {
       sessionID,
       lastIdx: newLastIdx,
       lastSignature: msg.new[sessionID].lastSignature,
-      bytesSinceLastSignature: newBytesSinceLastSignature,
+      bytesSinceLastSignature,
     };
 
     const sessionRowID: number = this.dbClient.addSessionUpdate({
@@ -339,5 +358,7 @@ export class StorageApiSync implements StorageAPI {
     return this.knwonStates.waitForSync(id, coValue);
   }
 
-  close() {}
+  close() {
+    return undefined;
+  }
 }

package/src/storage/syncUtils.ts

@@ -1,5 +1,9 @@
+import { getTransactionSize } from "../coValueContentMessage.js";
 import { getDependedOnCoValuesFromRawData } from "../coValueCore/utils.js";
-import type { CoValueHeader } from "../coValueCore/verifiedState.js";
+import type {
+  CoValueHeader,
+  Transaction,
+} from "../coValueCore/verifiedState.js";
 import type { Signature } from "../crypto/crypto.js";
 import type { SessionID } from "../exports.js";
 import type { NewContentMessage } from "../sync.js";
@@ -48,3 +52,7 @@ export function getDependedOnCoValues(
 
   return getDependedOnCoValuesFromRawData(id, header, sessionIDs, transactions);
 }
+
+export function getNewTransactionsSize(newTxs: Transaction[]) {
+  return newTxs.reduce((sum, tx) => sum + getTransactionSize(tx), 0);
+}

package/src/storage/types.ts

@@ -6,6 +6,10 @@ import { Signature } from "../crypto/crypto.js";
 import type { CoValueCore, RawCoID, SessionID } from "../exports.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 
+export type CorrectionCallback = (
+  correction: CoValueKnownState,
+) => NewContentMessage[] | undefined;
+
 /**
  * The StorageAPI is the interface that the StorageSync and StorageAsync classes implement.
  *
@@ -18,16 +22,13 @@ export interface StorageAPI {
     callback: (data: NewContentMessage) => void,
     done?: (found: boolean) => void,
   ): void;
-  store(
-    data: NewContentMessage[] | undefined,
-    handleCorrection: (correction: CoValueKnownState) => void,
-  ): void;
+  store(data: NewContentMessage, handleCorrection: CorrectionCallback): void;
 
   getKnownState(id: string): CoValueKnownState;
 
   waitForSync(id: string, coValue: CoValueCore): Promise<void>;
 
-  close(): void;
+  close(): Promise<unknown> | undefined;
 }
 
 export type CoValueRow = {

package/src/sync.ts

@@ -1,15 +1,24 @@
 import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
+import {
+  getTransactionSize,
+  knownStateFromContent,
+} from "./coValueContentMessage.js";
 import { CoValueCore } from "./coValueCore/coValueCore.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
-import { CoValueHeader, Transaction } from "./coValueCore/verifiedState.js";
+import {
+  CoValueHeader,
+  Transaction,
+  VerifiedState,
+} from "./coValueCore/verifiedState.js";
 import { Signature } from "./crypto/crypto.js";
-import { RawCoID, SessionID } from "./ids.js";
+import { RawCoID, SessionID, isRawCoID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
 import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
+import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "./typeUtils/isAccountID.js";
 
@@ -57,10 +66,12 @@ export type NewContentMessage = {
 };
 
 export type SessionNewContent = {
+  // The index where to start appending the new transactions. The index counting starts from 1.
   after: number;
   newTransactions: Transaction[];
   lastSignature: Signature;
 };
+
 export type DoneMessage = {
   action: "done";
   id: RawCoID;
@@ -162,13 +173,9 @@ export class SyncManager {
   }
 
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
-    if (msg.id === undefined || msg.id === null) {
-      logger.warn("Received sync message with undefined id", {
-        msg,
-      });
-      return;
-    } else if (!msg.id.startsWith("co_z")) {
-      logger.warn("Received sync message with invalid id", {
+    if (!isRawCoID(msg.id)) {
+      const errorType = msg.id ? "invalid" : "undefined";
+      logger.warn(`Received sync message with ${errorType} id`, {
         msg,
       });
       return;
@@ -431,12 +438,9 @@ export class SyncManager {
 
   recordTransactionsSize(newTransactions: Transaction[], source: string) {
     for (const tx of newTransactions) {
-      const txLength =
-        tx.privacy === "private"
-          ? tx.encryptedChanges.length
-          : tx.changes.length;
+      const size = getTransactionSize(tx);
 
-      this.transactionsSizeHistogram.record(txLength, {
+      this.transactionsSizeHistogram.record(size, {
         source,
       });
     }
@@ -674,7 +678,7 @@ export class SyncManager {
     const syncedPeers = [];
 
     if (from !== "storage") {
-      this.storeCoValue(coValue, [msg]);
+      this.storeContent(msg);
     }
 
     for (const peer of this.peersInPriorityOrder()) {
@@ -736,60 +740,18 @@ export class SyncManager {
     };
   }
 
-  requestedSyncs = new Set<RawCoID>();
-  requestCoValueSync(coValue: CoValueCore) {
-    if (this.requestedSyncs.has(coValue.id)) {
-      return;
-    }
-
-    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
-      trackingSet.add(coValue.id);
-    }
-
-    queueMicrotask(() => {
-      if (this.requestedSyncs.has(coValue.id)) {
-        this.syncCoValue(coValue);
-      }
-    });
-
-    this.requestedSyncs.add(coValue.id);
-  }
-
-  storeCoValue(coValue: CoValueCore, data: NewContentMessage[] | undefined) {
-    const storage = this.local.storage;
-
-    if (!storage || !data) return;
-
-    // Try to store the content as-is for performance
-    // In case that some transactions are missing, a correction will be requested, but it's an edge case
-    storage.store(data, (correction) => {
-      if (!coValue.hasVerifiedContent()) return;
-
-      const newContentPieces = coValue.verified.newContentSince(correction);
-
-      if (!newContentPieces) return;
+  private syncQueue = new LocalTransactionsSyncQueue((content) =>
+    this.syncContent(content),
+  );
+  syncHeader = this.syncQueue.syncHeader;
+  syncLocalTransaction = this.syncQueue.syncTransaction;
 
-      storage.store(newContentPieces, (response) => {
-        logger.error(
-          "Correction requested by storage after sending a correction content",
-          {
-            response,
-            knownState: coValue.knownState(),
-          },
-        );
-      });
-    });
-  }
+  syncContent(content: NewContentMessage) {
+    const coValue = this.local.getCoValue(content.id);
 
-  syncCoValue(coValue: CoValueCore) {
-    this.requestedSyncs.delete(coValue.id);
+    this.storeContent(content);
 
-    if (this.local.storage && coValue.hasVerifiedContent()) {
-      const knownState = this.local.storage.getKnownState(coValue.id);
-      const newContentPieces = coValue.verified.newContentSince(knownState);
-
-      this.storeCoValue(coValue, newContentPieces);
-    }
+    const contentKnownState = knownStateFromContent(content);
 
     for (const peer of this.peersInPriorityOrder()) {
       if (peer.closed) continue;
@@ -803,7 +765,11 @@ export class SyncManager {
         continue;
       }
 
-      this.sendNewContentIncludingDependencies(coValue.id, peer);
+      // We assume that the peer already knows anything before this content
+      // Any eventual reconciliation will be handled through the known state messages exchange
+      this.trySendToPeer(peer, content);
+      peer.combineOptimisticWith(coValue.id, contentKnownState);
+      peer.trackToldKnownState(coValue.id);
     }
 
     for (const peer of this.getPeers()) {
@@ -811,6 +777,20 @@ export class SyncManager {
     }
   }
 
+  private storeContent(content: NewContentMessage) {
+    const storage = this.local.storage;
+
+    if (!storage) return;
+
+    // Try to store the content as-is for performance
+    // In case that some transactions are missing, a correction will be requested, but it's an edge case
+    storage.store(content, (correction) => {
+      return this.local
+        .getCoValue(content.id)
+        .verified?.newContentSince(correction);
+    });
+  }
+
   waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const { syncState } = this;
     const currentSyncState = syncState.getCurrentSyncState(peerId, id);