cojson 0.18.33 → 0.18.35

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -1,17 +1,8 @@
-import {
-  addTransactionToContentMessage,
-  createContentMessage,
-  knownStateFromContent,
-} from "../coValueContentMessage.js";
-import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
-import { Signature } from "../crypto/crypto.js";
-import { RawCoID, SessionID } from "../ids.js";
-import {
-  combineKnownStateSessions,
-  KnownStateSessions,
-} from "../knownState.js";
+import { knownStateFromContent } from "../coValueContentMessage.js";
+import { VerifiedState } from "../coValueCore/verifiedState.js";
+import { RawCoID } from "../ids.js";
+import { combineKnownStateSessions, CoValueKnownState } from "../knownState.js";
 import { NewContentMessage } from "../sync.js";
-import { LinkedList } from "./LinkedList.js";
 
 /**
  * This queue is used to batch the sync of local transactions while preserving the order of updates between CoValues.
@@ -23,65 +14,97 @@ import { LinkedList } from "./LinkedList.js";
  * 2. If we do multiple updates on the same CoMap, the updates will be batched because it's safe to do so.
  */
 export class LocalTransactionsSyncQueue {
-  private readonly queue = new LinkedList<NewContentMessage>();
+  private batch: NewContentMessage[] = [];
+  private firstChunks = new Map<RawCoID, NewContentMessage>();
+  private lastUpdatedValue: VerifiedState | undefined;
+  private lastUpdatedValueKnownState: CoValueKnownState | undefined;
 
   constructor(private readonly sync: (content: NewContentMessage) => void) {}
 
-  syncHeader = (coValue: VerifiedState) => {
-    const lastPendingSync = this.queue.tail?.value;
+  syncTransaction = (
+    coValue: VerifiedState,
+    knownStateBefore: CoValueKnownState,
+  ) => {
+    const lastUpdatedValue = this.lastUpdatedValue;
+    const lastUpdatedValueKnownState = this.lastUpdatedValueKnownState;
 
-    if (lastPendingSync?.id === coValue.id) {
-      return;
+    if (lastUpdatedValue && lastUpdatedValueKnownState) {
+      if (lastUpdatedValue.id === coValue.id) {
+        return;
+      }
+
+      this.addContentToBatch(lastUpdatedValue, lastUpdatedValueKnownState);
+    }
+
+    this.lastUpdatedValue = coValue;
+    this.lastUpdatedValueKnownState = knownStateBefore;
+
+    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
+      trackingSet.add(coValue.id);
     }
 
-    this.enqueue(createContentMessage(coValue.id, coValue.header));
+    this.scheduleNextBatch();
   };
 
-  syncTransaction = (
+  private addContentToBatch(
     coValue: VerifiedState,
-    transaction: Transaction,
-    sessionID: SessionID,
-    signature: Signature,
-    txIdx: number,
-  ) => {
-    const lastPendingSync = this.queue.tail?.value;
-    const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
-    const isSignatureCheckpoint =
-      lastSignatureIdx > -1 && lastSignatureIdx === txIdx;
-
-    if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
-      addTransactionToContentMessage(
-        lastPendingSync,
-        transaction,
-        sessionID,
-        signature,
-        txIdx,
-      );
+    knownStateBefore: CoValueKnownState,
+  ) {
+    const content = coValue.newContentSince(knownStateBefore, {
+      skipExpectContentUntil: true, // we need to calculate the streaming header considering the current batch
+    });
 
+    if (!content) {
       return;
     }
 
-    const content = createContentMessage(coValue.id, coValue.header, false);
+    let firstChunk = this.firstChunks.get(coValue.id);
 
-    addTransactionToContentMessage(
-      content,
-      transaction,
-      sessionID,
-      signature,
-      txIdx,
-    );
+    for (const piece of content) {
+      this.batch.push(piece);
 
-    this.enqueue(content);
-  };
+      // Check if the local content updates are in streaming, if so we need to add the info to the first chunk
+      if (firstChunk) {
+        if (!firstChunk.expectContentUntil) {
+          firstChunk.expectContentUntil =
+            knownStateFromContent(firstChunk).sessions;
+        }
+        combineKnownStateSessions(
+          firstChunk.expectContentUntil,
+          knownStateFromContent(piece).sessions,
+        );
+      } else {
+        firstChunk = piece;
+        this.firstChunks.set(coValue.id, firstChunk);
+      }
+    }
+  }
 
-  enqueue(content: NewContentMessage) {
-    this.queue.push(content);
+  private nextBatchScheduled = false;
+  scheduleNextBatch() {
+    if (this.nextBatchScheduled) return;
 
-    this.processPendingSyncs();
+    this.nextBatchScheduled = true;
 
-    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
-      trackingSet.add(content.id);
-    }
+    queueMicrotask(() => {
+      if (this.lastUpdatedValue && this.lastUpdatedValueKnownState) {
+        this.addContentToBatch(
+          this.lastUpdatedValue,
+          this.lastUpdatedValueKnownState,
+        );
+      }
+      const batch = this.batch;
+
+      this.lastUpdatedValue = undefined;
+      this.lastUpdatedValueKnownState = undefined;
+      this.firstChunks = new Map();
+      this.batch = [];
+      this.nextBatchScheduled = false;
+
+      for (const content of batch) {
+        this.sync(content);
+      }
+    });
   }
 
   private dirtyCoValuesTrackingSets: Set<Set<RawCoID>> = new Set();
@@ -110,43 +133,4 @@ export class LocalTransactionsSyncQueue {
       },
     };
   };
-
-  private processingSyncs = false;
-  processPendingSyncs() {
-    if (this.processingSyncs) return;
-
-    this.processingSyncs = true;
-
-    queueMicrotask(() => {
-      const firstContentPieceMap = new Map<RawCoID, NewContentMessage>();
-
-      while (this.queue.head) {
-        const content = this.queue.head.value;
-
-        const firstContentPiece = firstContentPieceMap.get(content.id);
-
-        if (!firstContentPiece) {
-          firstContentPieceMap.set(content.id, content);
-        } else {
-          // There is already a content piece for this coValue, so this means that we need to flag
-          // that this content is going to be streamed
-          if (!firstContentPiece.expectContentUntil) {
-            firstContentPiece.expectContentUntil =
-              knownStateFromContent(firstContentPiece).sessions;
-          }
-
-          combineKnownStateSessions(
-            firstContentPiece.expectContentUntil,
-            knownStateFromContent(content).sessions,
-          );
-        }
-
-        this.sync(content);
-
-        this.queue.shift();
-      }
-
-      this.processingSyncs = false;
-    });
-  }
 }

package/src/storage/knownState.ts

@@ -3,7 +3,7 @@ import { RawCoID } from "../ids.js";
 import {
   CoValueKnownState,
   emptyKnownState,
-  isKnownStateSubsetOf,
+  areCurrentSessionsInSyncWith,
 } from "../knownState.js";
 
 /**
@@ -87,7 +87,7 @@ function isInSync(
     return false;
   }
 
-  return isKnownStateSubsetOf(
+  return areCurrentSessionsInSyncWith(
     knownState.sessions,
     knownStateFromStorage.sessions,
   );

package/src/sync.ts

@@ -4,6 +4,8 @@ import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import {
   getContenDebugInfo,
+  getNewTransactionsFromContentMessage,
+  getSessionEntriesFromContentMessage,
   getTransactionSize,
   knownStateFromContent,
 } from "./coValueContentMessage.js";
@@ -609,56 +611,55 @@ export class SyncManager {
     /**
      * The coValue is in memory, load the transactions from the content message
      */
-    for (const [sessionID, newContentForSession] of Object.entries(msg.new) as [
-      SessionID,
-      SessionNewContent,
-    ][]) {
-      const ourKnownTxIdx =
-        coValue.verified.sessions.get(sessionID)?.transactions.length;
-      const theirFirstNewTxIdx = newContentForSession.after;
-
-      if ((ourKnownTxIdx || 0) < theirFirstNewTxIdx) {
+    for (const [
+      sessionID,
+      newContentForSession,
+    ] of getSessionEntriesFromContentMessage(msg)) {
+      const newTransactions = getNewTransactionsFromContentMessage(
+        newContentForSession,
+        coValue.knownState(),
+        sessionID,
+      );
+
+      if (newTransactions === undefined) {
         invalidStateAssumed = true;
         continue;
       }
 
-      const alreadyKnownOffset = ourKnownTxIdx
-        ? ourKnownTxIdx - theirFirstNewTxIdx
-        : 0;
-
-      const newTransactions =
-        newContentForSession.newTransactions.slice(alreadyKnownOffset);
-
       if (newTransactions.length === 0) {
         continue;
       }
 
       // TODO: Handle invalid signatures in the middle of streaming
       // This could cause a situation where we are unable to load a chunk, and ask for a correction for all the subsequent chunks
-      const result = coValue.tryAddTransactions(
+      const error = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
         newContentForSession.lastSignature,
         this.skipVerify,
       );
 
-      if (result.isErr()) {
+      if (error) {
         if (peer) {
           logger.error("Failed to add transactions", {
             peerId: peer.id,
             peerRole: peer.role,
             id: msg.id,
-            err: result.error,
+            errorType: error.type,
+            err: error.error,
+            sessionID,
             msgKnownState: knownStateFromContent(msg).sessions,
+            msgSummary: getContenDebugInfo(msg),
             knownState: coValue.knownState().sessions,
-            newContent: validNewContent.new,
           });
           // TODO Mark only the session as errored, not the whole coValue
-          coValue.markErrored(peer.id, result.error);
+          coValue.markErrored(peer.id, error);
         } else {
           logger.error("Failed to add transactions from storage", {
             id: msg.id,
-            err: result.error,
+            err: error.error,
+            sessionID,
+            errorType: error.type,
           });
         }
         continue;
@@ -669,9 +670,7 @@ export class SyncManager {
       }
 
       // The new content for this session has been verified, so we can store it
-      if (result.value) {
-        validNewContent.new[sessionID] = newContentForSession;
-      }
+      validNewContent.new[sessionID] = newContentForSession;
     }
 
     if (peer) {
@@ -771,7 +770,6 @@ export class SyncManager {
   private syncQueue = new LocalTransactionsSyncQueue((content) =>
     this.syncContent(content),
   );
-  syncHeader = this.syncQueue.syncHeader;
   syncLocalTransaction = this.syncQueue.syncTransaction;
   trackDirtyCoValues = this.syncQueue.trackDirtyCoValues;
 

package/src/tests/coPlainText.test.ts

@@ -371,13 +371,12 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
     }),
   ).toMatchInlineSnapshot(`
     [
-      "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1 expectContentUntil: header/42",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 1 New: 1",
+      "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 2 expectContentUntil: header/42",
       "client -> storage | CONTENT CoPlainText header: false new: After: 2 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 3 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 4 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 5 New: 2",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 7 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 5 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 6 New: 2",
       "client -> storage | CONTENT CoPlainText header: false new: After: 8 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 9 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 10 New: 1",
@@ -389,8 +388,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "client -> storage | CONTENT CoPlainText header: false new: After: 16 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 17 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 18 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 19 New: 2",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 21 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 19 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 20 New: 2",
       "client -> storage | CONTENT CoPlainText header: false new: After: 22 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 23 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 24 New: 1",
@@ -406,16 +405,16 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "client -> storage | CONTENT CoPlainText header: false new: After: 34 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 35 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 36 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 37 New: 3",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 40 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 37 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 38 New: 3",
       "client -> storage | CONTENT CoPlainText header: false new: After: 41 New: 1",
       "client -> storage | LOAD CoPlainText sessions: empty",
       "storage -> client | CONTENT CoPlainText header: true new: After: 0 New: 2 expectContentUntil: header/42",
       "storage -> client | CONTENT CoPlainText header: true new: After: 2 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 3 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 4 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 5 New: 2",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 7 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 5 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 6 New: 2",
       "storage -> client | CONTENT CoPlainText header: true new: After: 8 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 9 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 10 New: 1",
@@ -427,8 +426,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "storage -> client | CONTENT CoPlainText header: true new: After: 16 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 17 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 18 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 19 New: 2",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 21 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 19 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 20 New: 2",
       "storage -> client | CONTENT CoPlainText header: true new: After: 22 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 23 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 24 New: 1",
@@ -444,8 +443,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "storage -> client | CONTENT CoPlainText header: true new: After: 34 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 35 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 36 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 37 New: 3",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 40 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 37 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 38 New: 3",
       "storage -> client | CONTENT CoPlainText header: true new: After: 41 New: 1",
     ]
   `);

package/src/tests/coValueContentMessage.test.ts

@@ -1,10 +1,15 @@
 import { describe, expect, test } from "vitest";
-import { knownStateFromContent } from "../coValueContentMessage.js";
+import {
+  getNewTransactionsFromContentMessage,
+  knownStateFromContent,
+} from "../coValueContentMessage.js";
 import { emptyKnownState } from "../knownState.js";
-import { NewContentMessage } from "../sync.js";
+import { NewContentMessage, SessionNewContent } from "../sync.js";
 import type { RawCoID, SessionID } from "../ids.js";
 import { stableStringify } from "../jsonStringify.js";
 import { CO_VALUE_PRIORITY } from "../priority.js";
+import { CoValueKnownState } from "../knownState.js";
+import { Transaction } from "../coValueCore/verifiedState.js";
 
 describe("knownStateFromContent", () => {
   const mockCoID: RawCoID = "co_z1234567890abcdef";
@@ -213,3 +218,193 @@ describe("knownStateFromContent", () => {
     expect(result.sessions[mockSessionID1]).toBe(1); // 0 + 1
   });
 });
+
+describe("getNewTransactionsFromContentMessage", () => {
+  const mockCoID: RawCoID = "co_z1234567890abcdef";
+  const mockSessionID: SessionID = "sealer_z123/signer_z456_session_z789";
+
+  function createTransaction(): Transaction {
+    return {
+      privacy: "trusting",
+      madeAt: Date.now(),
+      changes: stableStringify([{ op: "set", key: "test", value: "value" }]),
+    };
+  }
+
+  function createKnownState(
+    sessionTxIdx: number | undefined,
+  ): CoValueKnownState {
+    const knownState = emptyKnownState(mockCoID);
+    knownState.header = true;
+    if (sessionTxIdx !== undefined) {
+      knownState.sessions[mockSessionID] = sessionTxIdx;
+    }
+    return knownState;
+  }
+
+  test("returns all transactions when we know none (ourKnownTxIdx = 0, theirFirstNewTxIdx = 0)", () => {
+    const transactions = [createTransaction(), createTransaction()];
+    const content: SessionNewContent = {
+      after: 0,
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(undefined); // defaults to 0
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual(transactions);
+    expect(result).toHaveLength(2);
+  });
+
+  test("returns subset of transactions when we know some (ourKnownTxIdx = 3, theirFirstNewTxIdx = 0)", () => {
+    const transactions = [
+      createTransaction(),
+      createTransaction(),
+      createTransaction(),
+      createTransaction(),
+      createTransaction(),
+    ];
+    const content: SessionNewContent = {
+      after: 0,
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(3); // we already know txs 0, 1, 2
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual(transactions.slice(3));
+    expect(result).toHaveLength(2);
+  });
+
+  test("returns undefined when we're missing transactions (ourKnownTxIdx = 2, theirFirstNewTxIdx = 5)", () => {
+    const transactions = [createTransaction(), createTransaction()];
+    const content: SessionNewContent = {
+      after: 5, // they're sending txs starting from idx 5
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(2); // but we only know up to idx 2
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toBeUndefined();
+  });
+
+  test("returns empty array when we know all transactions (ourKnownTxIdx = 5, theirFirstNewTxIdx = 2)", () => {
+    const transactions = [
+      createTransaction(),
+      createTransaction(),
+      createTransaction(),
+    ];
+    const content: SessionNewContent = {
+      after: 2, // they're sending txs 2, 3, 4
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(5); // we already know txs up to idx 5
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual([]);
+    expect(result).toHaveLength(0);
+  });
+
+  test("returns all transactions when ourKnownTxIdx equals theirFirstNewTxIdx", () => {
+    const transactions = [createTransaction(), createTransaction()];
+    const content: SessionNewContent = {
+      after: 3,
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(3); // we know up to idx 3
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual(transactions);
+    expect(result).toHaveLength(2);
+  });
+
+  test("handles session not in knownState (defaults to 0)", () => {
+    const transactions = [createTransaction()];
+    const content: SessionNewContent = {
+      after: 1,
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = emptyKnownState(mockCoID);
+    knownState.header = true;
+    // no sessions defined, so mockSessionID defaults to 0
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    // ourKnownTxIdx = 0, theirFirstNewTxIdx = 1, so 0 < 1 -> undefined
+    expect(result).toBeUndefined();
+  });
+
+  test("returns single transaction when offset is length - 1", () => {
+    const transactions = [
+      createTransaction(),
+      createTransaction(),
+      createTransaction(),
+    ];
+    const content: SessionNewContent = {
+      after: 5,
+      newTransactions: transactions,
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(7); // we know up to idx 7 (5 + 2 transactions)
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual([transactions[2]]);
+    expect(result).toHaveLength(1);
+  });
+
+  test("handles empty newTransactions array", () => {
+    const content: SessionNewContent = {
+      after: 5,
+      newTransactions: [],
+      lastSignature: "signature_z1234",
+    };
+    const knownState = createKnownState(5);
+
+    const result = getNewTransactionsFromContentMessage(
+      content,
+      knownState,
+      mockSessionID,
+    );
+
+    expect(result).toEqual([]);
+    expect(result).toHaveLength(0);
+  });
+});

package/src/tests/coValueCore.isCompletelyDownloaded.test.ts

@@ -537,8 +537,9 @@ describe("CoValueCore.isCompletelyDownloaded", () => {
       bobSession.syncManager.handleNewContent(lastChunk, "import");
 
       // Wait for the notification to be scheduled and executed
-      await waitFor(() => mapOnBob.core.isCompletelyDownloaded());
-      expect(mapOnBob.core.isCompletelyDownloaded()).toBe(true);
+      await waitFor(() => {
+        expect(mapOnBob.core.isCompletelyDownloaded()).toBe(true);
+      });
     });
 
     test.skip("should return false when the owner of the value is streaming", async () => {