cojson 0.18.33 → 0.18.34

package/src/knownState.ts

@@ -106,18 +106,62 @@ export function cloneKnownState(knownState: CoValueKnownState) {
   };
 }
 
+/**
+ * Checks if all the local sessions have the same counters as in remote.
+ */
+export function areCurrentSessionsInSyncWith(
+  current: Record<string, number>,
+  target: Record<string, number>,
+) {
+  for (const [sessionId, currentCount] of Object.entries(current) as [
+    SessionID,
+    number,
+  ][]) {
+    const targetCount = target[sessionId] ?? 0;
+    if (currentCount !== targetCount) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
 /**
  * Checks if all the local sessions have the same counters as in remote.
  */
 export function isKnownStateSubsetOf(
-  local: Record<string, number>,
-  remote: Record<string, number>,
+  current: Record<string, number>,
+  target: Record<string, number>,
 ) {
-  for (const sessionId of Object.keys(local)) {
-    if (local[sessionId] !== remote[sessionId]) {
+  for (const [sessionId, currentCount] of Object.entries(current) as [
+    SessionID,
+    number,
+  ][]) {
+    const targetCount = target[sessionId] ?? 0;
+    if (currentCount > targetCount) {
       return false;
     }
   }
 
   return true;
 }
+
+/**
+ * Returns the record with the sessions that need to be sent to the target
+ */
+export function getKnownStateToSend(
+  current: Record<string, number>,
+  target: Record<string, number>,
+) {
+  const toSend: Record<string, number> = {};
+  for (const [sessionId, currentCount] of Object.entries(current) as [
+    SessionID,
+    number,
+  ][]) {
+    const targetCount = target[sessionId] ?? 0;
+    if (currentCount > targetCount) {
+      toSend[sessionId] = currentCount;
+    }
+  }
+  return toSend;
+}

package/src/localNode.ts

@@ -10,7 +10,6 @@ import {
 import {
   type CoValueHeader,
   type CoValueUniqueness,
-  VerifiedState,
 } from "./coValueCore/verifiedState.js";
 import {
   AccountMeta,
@@ -31,7 +30,7 @@ import {
   type RawGroup,
   secretSeedFromInviteSecret,
 } from "./coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG, GARBAGE_COLLECTOR_CONFIG } from "./config.js";
+import { CO_VALUE_LOADING_CONFIG } from "./config.js";
 import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { logger } from "./logger.js";
@@ -41,6 +40,7 @@ import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromS
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
 import { connectedPeers } from "./streamUtils.js";
+import { emptyKnownState } from "./knownState.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
@@ -380,7 +380,10 @@ export class LocalNode {
     }
 
     this.garbageCollector?.trackCoValueAccess(coValue);
-    this.syncManager.syncHeader(coValue.verified);
+    this.syncManager.syncLocalTransaction(
+      coValue.verified,
+      emptyKnownState(id),
+    );
 
     return coValue;
   }

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -1,17 +1,8 @@
-import {
-  addTransactionToContentMessage,
-  createContentMessage,
-  knownStateFromContent,
-} from "../coValueContentMessage.js";
-import { Transaction, VerifiedState } from "../coValueCore/verifiedState.js";
-import { Signature } from "../crypto/crypto.js";
-import { RawCoID, SessionID } from "../ids.js";
-import {
-  combineKnownStateSessions,
-  KnownStateSessions,
-} from "../knownState.js";
+import { knownStateFromContent } from "../coValueContentMessage.js";
+import { VerifiedState } from "../coValueCore/verifiedState.js";
+import { RawCoID } from "../ids.js";
+import { combineKnownStateSessions, CoValueKnownState } from "../knownState.js";
 import { NewContentMessage } from "../sync.js";
-import { LinkedList } from "./LinkedList.js";
 
 /**
  * This queue is used to batch the sync of local transactions while preserving the order of updates between CoValues.
@@ -23,65 +14,97 @@ import { LinkedList } from "./LinkedList.js";
  * 2. If we do multiple updates on the same CoMap, the updates will be batched because it's safe to do so.
  */
 export class LocalTransactionsSyncQueue {
-  private readonly queue = new LinkedList<NewContentMessage>();
+  private batch: NewContentMessage[] = [];
+  private firstChunks = new Map<RawCoID, NewContentMessage>();
+  private lastUpdatedValue: VerifiedState | undefined;
+  private lastUpdatedValueKnownState: CoValueKnownState | undefined;
 
   constructor(private readonly sync: (content: NewContentMessage) => void) {}
 
-  syncHeader = (coValue: VerifiedState) => {
-    const lastPendingSync = this.queue.tail?.value;
+  syncTransaction = (
+    coValue: VerifiedState,
+    knownStateBefore: CoValueKnownState,
+  ) => {
+    const lastUpdatedValue = this.lastUpdatedValue;
+    const lastUpdatedValueKnownState = this.lastUpdatedValueKnownState;
 
-    if (lastPendingSync?.id === coValue.id) {
-      return;
+    if (lastUpdatedValue && lastUpdatedValueKnownState) {
+      if (lastUpdatedValue.id === coValue.id) {
+        return;
+      }
+
+      this.addContentToBatch(lastUpdatedValue, lastUpdatedValueKnownState);
+    }
+
+    this.lastUpdatedValue = coValue;
+    this.lastUpdatedValueKnownState = knownStateBefore;
+
+    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
+      trackingSet.add(coValue.id);
     }
 
-    this.enqueue(createContentMessage(coValue.id, coValue.header));
+    this.scheduleNextBatch();
   };
 
-  syncTransaction = (
+  private addContentToBatch(
     coValue: VerifiedState,
-    transaction: Transaction,
-    sessionID: SessionID,
-    signature: Signature,
-    txIdx: number,
-  ) => {
-    const lastPendingSync = this.queue.tail?.value;
-    const lastSignatureIdx = coValue.getLastSignatureCheckpoint(sessionID);
-    const isSignatureCheckpoint =
-      lastSignatureIdx > -1 && lastSignatureIdx === txIdx;
-
-    if (lastPendingSync?.id === coValue.id && !isSignatureCheckpoint) {
-      addTransactionToContentMessage(
-        lastPendingSync,
-        transaction,
-        sessionID,
-        signature,
-        txIdx,
-      );
+    knownStateBefore: CoValueKnownState,
+  ) {
+    const content = coValue.newContentSince(knownStateBefore, {
+      skipExpectContentUntil: true, // we need to calculate the streaming header considering the current batch
+    });
 
+    if (!content) {
       return;
     }
 
-    const content = createContentMessage(coValue.id, coValue.header, false);
+    let firstChunk = this.firstChunks.get(coValue.id);
 
-    addTransactionToContentMessage(
-      content,
-      transaction,
-      sessionID,
-      signature,
-      txIdx,
-    );
+    for (const piece of content) {
+      this.batch.push(piece);
 
-    this.enqueue(content);
-  };
+      // Check if the local content updates are in streaming, if so we need to add the info to the first chunk
+      if (firstChunk) {
+        if (!firstChunk.expectContentUntil) {
+          firstChunk.expectContentUntil =
+            knownStateFromContent(firstChunk).sessions;
+        }
+        combineKnownStateSessions(
+          firstChunk.expectContentUntil,
+          knownStateFromContent(piece).sessions,
+        );
+      } else {
+        firstChunk = piece;
+        this.firstChunks.set(coValue.id, firstChunk);
+      }
+    }
+  }
 
-  enqueue(content: NewContentMessage) {
-    this.queue.push(content);
+  private nextBatchScheduled = false;
+  scheduleNextBatch() {
+    if (this.nextBatchScheduled) return;
 
-    this.processPendingSyncs();
+    this.nextBatchScheduled = true;
 
-    for (const trackingSet of this.dirtyCoValuesTrackingSets) {
-      trackingSet.add(content.id);
-    }
+    queueMicrotask(() => {
+      if (this.lastUpdatedValue && this.lastUpdatedValueKnownState) {
+        this.addContentToBatch(
+          this.lastUpdatedValue,
+          this.lastUpdatedValueKnownState,
+        );
+      }
+      const batch = this.batch;
+
+      this.lastUpdatedValue = undefined;
+      this.lastUpdatedValueKnownState = undefined;
+      this.firstChunks = new Map();
+      this.batch = [];
+      this.nextBatchScheduled = false;
+
+      for (const content of batch) {
+        this.sync(content);
+      }
+    });
   }
 
   private dirtyCoValuesTrackingSets: Set<Set<RawCoID>> = new Set();
@@ -110,43 +133,4 @@ export class LocalTransactionsSyncQueue {
       },
     };
   };
-
-  private processingSyncs = false;
-  processPendingSyncs() {
-    if (this.processingSyncs) return;
-
-    this.processingSyncs = true;
-
-    queueMicrotask(() => {
-      const firstContentPieceMap = new Map<RawCoID, NewContentMessage>();
-
-      while (this.queue.head) {
-        const content = this.queue.head.value;
-
-        const firstContentPiece = firstContentPieceMap.get(content.id);
-
-        if (!firstContentPiece) {
-          firstContentPieceMap.set(content.id, content);
-        } else {
-          // There is already a content piece for this coValue, so this means that we need to flag
-          // that this content is going to be streamed
-          if (!firstContentPiece.expectContentUntil) {
-            firstContentPiece.expectContentUntil =
-              knownStateFromContent(firstContentPiece).sessions;
-          }
-
-          combineKnownStateSessions(
-            firstContentPiece.expectContentUntil,
-            knownStateFromContent(content).sessions,
-          );
-        }
-
-        this.sync(content);
-
-        this.queue.shift();
-      }
-
-      this.processingSyncs = false;
-    });
-  }
 }

package/src/storage/knownState.ts

@@ -3,7 +3,7 @@ import { RawCoID } from "../ids.js";
 import {
   CoValueKnownState,
   emptyKnownState,
-  isKnownStateSubsetOf,
+  areCurrentSessionsInSyncWith,
 } from "../knownState.js";
 
 /**
@@ -87,7 +87,7 @@ function isInSync(
     return false;
   }
 
-  return isKnownStateSubsetOf(
+  return areCurrentSessionsInSyncWith(
     knownState.sessions,
     knownStateFromStorage.sessions,
   );

package/src/sync.ts

@@ -771,7 +771,6 @@ export class SyncManager {
   private syncQueue = new LocalTransactionsSyncQueue((content) =>
     this.syncContent(content),
   );
-  syncHeader = this.syncQueue.syncHeader;
   syncLocalTransaction = this.syncQueue.syncTransaction;
   trackDirtyCoValues = this.syncQueue.trackDirtyCoValues;
 

package/src/tests/coPlainText.test.ts

@@ -371,13 +371,12 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
     }),
   ).toMatchInlineSnapshot(`
     [
-      "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 1 expectContentUntil: header/42",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 1 New: 1",
+      "client -> storage | CONTENT CoPlainText header: true new: After: 0 New: 2 expectContentUntil: header/42",
       "client -> storage | CONTENT CoPlainText header: false new: After: 2 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 3 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 4 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 5 New: 2",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 7 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 5 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 6 New: 2",
       "client -> storage | CONTENT CoPlainText header: false new: After: 8 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 9 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 10 New: 1",
@@ -389,8 +388,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "client -> storage | CONTENT CoPlainText header: false new: After: 16 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 17 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 18 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 19 New: 2",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 21 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 19 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 20 New: 2",
       "client -> storage | CONTENT CoPlainText header: false new: After: 22 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 23 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 24 New: 1",
@@ -406,16 +405,16 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "client -> storage | CONTENT CoPlainText header: false new: After: 34 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 35 New: 1",
       "client -> storage | CONTENT CoPlainText header: false new: After: 36 New: 1",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 37 New: 3",
-      "client -> storage | CONTENT CoPlainText header: false new: After: 40 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 37 New: 1",
+      "client -> storage | CONTENT CoPlainText header: false new: After: 38 New: 3",
       "client -> storage | CONTENT CoPlainText header: false new: After: 41 New: 1",
       "client -> storage | LOAD CoPlainText sessions: empty",
       "storage -> client | CONTENT CoPlainText header: true new: After: 0 New: 2 expectContentUntil: header/42",
       "storage -> client | CONTENT CoPlainText header: true new: After: 2 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 3 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 4 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 5 New: 2",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 7 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 5 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 6 New: 2",
       "storage -> client | CONTENT CoPlainText header: true new: After: 8 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 9 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 10 New: 1",
@@ -427,8 +426,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "storage -> client | CONTENT CoPlainText header: true new: After: 16 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 17 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 18 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 19 New: 2",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 21 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 19 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 20 New: 2",
       "storage -> client | CONTENT CoPlainText header: true new: After: 22 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 23 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 24 New: 1",
@@ -444,8 +443,8 @@ test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX
       "storage -> client | CONTENT CoPlainText header: true new: After: 34 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 35 New: 1",
       "storage -> client | CONTENT CoPlainText header: true new: After: 36 New: 1",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 37 New: 3",
-      "storage -> client | CONTENT CoPlainText header: true new: After: 40 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 37 New: 1",
+      "storage -> client | CONTENT CoPlainText header: true new: After: 38 New: 3",
       "storage -> client | CONTENT CoPlainText header: true new: After: 41 New: 1",
     ]
   `);

package/src/tests/coValueCore.isCompletelyDownloaded.test.ts

@@ -537,8 +537,9 @@ describe("CoValueCore.isCompletelyDownloaded", () => {
       bobSession.syncManager.handleNewContent(lastChunk, "import");
 
       // Wait for the notification to be scheduled and executed
-      await waitFor(() => mapOnBob.core.isCompletelyDownloaded());
-      expect(mapOnBob.core.isCompletelyDownloaded()).toBe(true);
+      await waitFor(() => {
+        expect(mapOnBob.core.isCompletelyDownloaded()).toBe(true);
+      });
     });
 
     test.skip("should return false when the owner of the value is streaming", async () => {

package/src/tests/coValueCore.isStreaming.test.ts

@@ -3,9 +3,12 @@ import {
   SyncMessagesLog,
   TEST_NODE_CONFIG,
   loadCoValueOrFail,
+  setupTestAccount,
   setupTestNode,
   waitFor,
 } from "./testUtils";
+import type { RawCoMap } from "../exports";
+import type { CoID } from "../coValue";
 
 let jazzCloud: ReturnType<typeof setupTestNode>;
 
@@ -82,7 +85,7 @@ describe("isStreaming", () => {
     expect(mapInNewSession.core.isStreaming()).toBe(false);
   });
 
-  test("loading a large content update  should be streaming until all chunks are sent", async () => {
+  test("loading a large content update should be streaming until all chunks are sent", async () => {
     const client = setupTestNode({
       connected: true,
     });
@@ -222,7 +225,8 @@ describe("isStreaming", () => {
     await map.core.waitForSync();
     const newSession = client.spawnNewSession();
 
-    await loadCoValueOrFail(newSession.node, map.id);
+    const mapInNewSession1 = await loadCoValueOrFail(newSession.node, map.id);
+    await mapInNewSession1.core.waitForFullStreaming();
 
     const content = map.core.verified.newContentSince(undefined);
     assert(content);
@@ -268,4 +272,82 @@ describe("isStreaming", () => {
 
     expect(mapInNewSession.core.isStreaming()).toBe(false);
   });
+
+  test("mixed updates should not leave isStreaming to true (3 sessions)", async () => {
+    const aliceLaptop = await setupTestAccount({
+      connected: true,
+    });
+
+    const group = aliceLaptop.node.createGroup();
+    const map = group.createMap();
+
+    map.set("count", 0, "trusting");
+    map.set("count", 1, "trusting");
+
+    await map.core.waitForSync();
+
+    const alicePhone = await aliceLaptop.spawnNewSession();
+
+    const mapOnPhone = await loadCoValueOrFail(alicePhone.node, map.id);
+
+    mapOnPhone.set("count", 2, "trusting");
+    mapOnPhone.set("count", 3, "trusting");
+    mapOnPhone.set("count", 4, "trusting");
+
+    await mapOnPhone.core.waitForSync();
+
+    const aliceTablet = await alicePhone.spawnNewSession();
+    const mapOnTablet = await loadCoValueOrFail(aliceTablet.node, map.id);
+
+    mapOnTablet.set("count", 5, "trusting");
+    mapOnTablet.set("count", 6, "trusting");
+    mapOnTablet.set("count", 7, "trusting");
+
+    await mapOnTablet.core.waitForSync();
+
+    map.set("count", 8, "trusting");
+    map.set("count", 9, "trusting");
+    map.set("count", 10, "trusting");
+
+    mapOnPhone.set("count", 11, "trusting");
+    mapOnTablet.set("count", 12, "trusting");
+
+    await map.core.waitForSync();
+    await mapOnPhone.core.waitForSync();
+    await mapOnTablet.core.waitForSync();
+
+    expect(map.core.isStreaming()).toBe(false);
+    expect(mapOnPhone.core.isStreaming()).toBe(false);
+    expect(mapOnTablet.core.isStreaming()).toBe(false);
+
+    const mapBranch = map.core.createBranch("test-branch");
+
+    const aliceTv = await aliceTablet.spawnNewSession();
+
+    const mapBranchOnTv = await loadCoValueOrFail(
+      aliceTv.node,
+      mapBranch.id as unknown as CoID<RawCoMap>,
+    );
+
+    mapBranchOnTv.set("count", 13, "trusting");
+    mapBranchOnTv.set("count", 14, "trusting");
+    mapBranchOnTv.set("count", 15, "trusting");
+
+    await mapBranchOnTv.core.waitForSync();
+
+    group.addMember("everyone", "reader");
+
+    const bob = await setupTestAccount({
+      connected: true,
+    });
+
+    const mapBranchOnBob = await loadCoValueOrFail(bob.node, mapBranchOnTv.id);
+
+    expect(mapBranchOnBob.core.isStreaming()).toBe(false);
+    expect(map.core.isStreaming()).toBe(false);
+    expect(mapOnPhone.core.isStreaming()).toBe(false);
+    expect(mapOnTablet.core.isStreaming()).toBe(false);
+
+    expect(mapBranchOnBob.get("count")).toBe(15);
+  });
 });

package/src/tests/group.childKeyRotation.test.ts

@@ -89,7 +89,7 @@ describe("Group.childKeyRotation", () => {
     const newBobSession = await bob.spawnNewSession();
 
     const childGroupOnNewBobNode = await loadCoValueOrFail(
-      newBobSession,
+      newBobSession.node,
       childGroup.id,
     );
 
@@ -138,15 +138,15 @@ describe("Group.childKeyRotation", () => {
     const newBobSession = await bob.spawnNewSession();
 
     for (const chunk of content) {
-      newBobSession.syncManager.handleNewContent(chunk, "import");
+      newBobSession.node.syncManager.handleNewContent(chunk, "import");
     }
 
     const childGroupOnNewBobNode = await loadCoValueOrFail(
-      newBobSession,
+      newBobSession.node,
       childGroup.id,
     );
 
-    newBobSession.syncManager.handleNewContent(lastChunk, "import");
+    newBobSession.node.syncManager.handleNewContent(lastChunk, "import");
 
     // The migration waits for the group to be completely downloaded
     await childGroupOnNewBobNode.core.waitForAsync((core) =>
@@ -205,15 +205,15 @@ describe("Group.childKeyRotation", () => {
     const newBobSession = await bob.spawnNewSession();
 
     for (const chunk of content) {
-      newBobSession.syncManager.handleNewContent(chunk, "import");
+      newBobSession.node.syncManager.handleNewContent(chunk, "import");
     }
 
     const childGroupOnNewBobNode = await loadCoValueOrFail(
-      newBobSession,
+      newBobSession.node,
       childGroup.id,
     );
 
-    newBobSession.syncManager.handleNewContent(lastChunk, "import");
+    newBobSession.node.syncManager.handleNewContent(lastChunk, "import");
 
     // The migration waits for the group to be completely downloaded, this includes full streaming of the parent group
     await childGroupOnNewBobNode.core.waitForAsync((core) =>
@@ -268,7 +268,7 @@ describe("Group.childKeyRotation", () => {
 
     // Instead Bob is an admin, so when loading the child group he can rotate the readKey
     const newBobSession = await bob.spawnNewSession();
-    const mapOnNewBobNode = await loadCoValueOrFail(newBobSession, map.id);
+    const mapOnNewBobNode = await loadCoValueOrFail(newBobSession.node, map.id);
 
     mapOnNewBobNode.set("test", "Not readable by charlie");
 
@@ -416,7 +416,7 @@ describe("Group.childKeyRotation", () => {
 
     const newBobSession = await bob.spawnNewSession();
     const childGroupOnNewBobNode = await loadCoValueOrFail(
-      newBobSession,
+      newBobSession.node,
       childGroup.id,
     );