cojson 0.18.19 → 0.18.20

package/src/localNode.ts

@@ -40,6 +40,7 @@ import { Peer, PeerID, SyncManager } from "./sync.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { canBeBranched } from "./coValueCore/branching.js";
+import { connectedPeers } from "./streamUtils.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
@@ -81,12 +82,15 @@ export class LocalNode {
     this.crypto = crypto;
   }
 
-  enableGarbageCollector() {
+  enableGarbageCollector(opts?: { garbageCollectGroups?: boolean }) {
     if (this.garbageCollector) {
       return;
     }
 
-    this.garbageCollector = new GarbageCollector(this.coValues);
+    this.garbageCollector = new GarbageCollector(
+      this.coValues,
+      opts?.garbageCollectGroups ?? false,
+    );
   }
 
   setStorage(storage: StorageAPI) {
@@ -99,7 +103,13 @@ export class LocalNode {
   }
 
   hasCoValue(id: RawCoID) {
-    return this.coValues.has(id);
+    const coValue = this.coValues.get(id);
+
+    if (!coValue) {
+      return false;
+    }
+
+    return coValue.loadingState !== "unknown";
   }
 
   getCoValue(id: RawCoID) {
@@ -119,16 +129,6 @@ export class LocalNode {
     return this.coValues.values();
   }
 
-  private putCoValue(
-    id: RawCoID,
-    verified: VerifiedState,
-    { forceOverwrite = false }: { forceOverwrite?: boolean } = {},
-  ): AvailableCoValueCore {
-    const entry = this.getCoValue(id);
-    entry.internalMarkMagicallyAvailable(verified, { forceOverwrite });
-    return entry as AvailableCoValueCore;
-  }
-
   internalDeleteCoValue(id: RawCoID) {
     this.coValues.delete(id);
   }
@@ -371,10 +371,13 @@ export class LocalNode {
 
     const id = idforHeader(header, this.crypto);
 
-    const coValue = this.putCoValue(
-      id,
-      new VerifiedState(id, this.crypto, header),
-    );
+    const coValue = this.getCoValue(id);
+
+    coValue.provideHeader(header);
+
+    if (!coValue.hasVerifiedContent()) {
+      throw new Error("CoValue not available after providing header");
+    }
 
     this.garbageCollector?.trackCoValueAccess(coValue);
     this.syncManager.syncHeader(coValue.verified);
@@ -624,11 +627,13 @@ export class LocalNode {
       return;
     }
 
+    const groupCoreAsDifferentAgent = await this.loadCoValueAsDifferentAgent(
+      group.id,
+      inviteAgentSecret,
+    );
+
     const groupAsInvite = expectGroup(
-      this.loadCoValueAsDifferentAgent(
-        group.id,
-        inviteAgentSecret,
-      ).getCurrentContent(),
+      groupCoreAsDifferentAgent.getCurrentContent(),
     );
 
     groupAsInvite.addMemberInternal(
@@ -755,7 +760,7 @@ export class LocalNode {
     return group;
   }
 
-  loadCoValueAsDifferentAgent(
+  async loadCoValueAsDifferentAgent(
     id: RawCoID,
     secret: AgentSecret,
     accountId?: RawAccountID | AgentID,
@@ -768,44 +773,34 @@ export class LocalNode {
       this.crypto,
     );
 
-    newNode.cloneVerifiedStateFrom(this, id);
+    await newNode.loadVerifiedStateFrom(this, id);
 
     return newNode.expectCoValueLoaded(id);
   }
 
   /** @internal */
-  cloneVerifiedStateFrom(otherNode: LocalNode, id: RawCoID) {
-    const coValuesIdsToCopy = [id];
+  async loadVerifiedStateFrom(otherNode: LocalNode, id: RawCoID) {
+    const connection = connectedPeers("source-" + id, "target-" + id, {
+      peer1role: "server",
+      peer2role: "client",
+    });
 
-    // Scan all the dependencies and add them to the list
-    for (let i = 0; i < coValuesIdsToCopy.length; i++) {
-      const coValueID = coValuesIdsToCopy[i]!;
-      const coValue = otherNode.getCoValue(coValueID);
+    this.syncManager.addPeer(connection[0], true);
+    otherNode.syncManager.addPeer(connection[1], true);
 
-      if (!coValue.isAvailable()) {
-        continue;
-      }
+    const coValue = this.getCoValue(id);
 
-      for (const dep of coValue.getDependedOnCoValues()) {
-        coValuesIdsToCopy.push(dep);
-      }
+    const peerState = this.syncManager.peers[connection[0].id];
+
+    if (!peerState) {
+      throw new Error("Peer state not found");
     }
 
-    // Copy the coValue all the dependencies by following the dependency order
-    while (coValuesIdsToCopy.length > 0) {
-      const coValueID = coValuesIdsToCopy.pop()!;
-      const coValue = otherNode.getCoValue(coValueID);
+    coValue.loadFromPeers([peerState]);
 
-      if (!coValue.isAvailable()) {
-        continue;
-      }
+    await coValue.waitForAvailable();
 
-      if (this.coValues.get(coValueID)?.isAvailable()) {
-        continue;
-      }
-
-      this.putCoValue(coValueID, coValue.verified);
-    }
+    peerState.gracefulShutdown();
   }
 
   /**

package/src/sync.ts

@@ -8,7 +8,6 @@ import {
   knownStateFromContent,
 } from "./coValueContentMessage.js";
 import { CoValueCore } from "./coValueCore/coValueCore.js";
-import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
 import { CoValueHeader, Transaction } from "./coValueCore/verifiedState.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID, isRawCoID } from "./ids.js";
@@ -17,8 +16,6 @@ import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
-import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
-import { isAccountID } from "./typeUtils/isAccountID.js";
 
 export type CoValueKnownState = {
   id: RawCoID;
@@ -463,7 +460,6 @@ export class SyncManager {
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
-        this.sendNewContent(msg.id, peer);
         return;
       }
 
@@ -524,8 +520,42 @@ export class SyncManager {
           ? "import"
           : peer?.role;
 
+    coValue.addDependenciesFromContentMessage(msg);
+
+    // If some of the dependencies are missing, we wait for them to be available
+    // before handling the new content
+    // This must happen even if the dependencies are not related to this content
+    // but the content we've got before
+    if (!this.skipVerify && coValue.hasMissingDependencies()) {
+      coValue.addNewContentToQueue(msg, from);
+
+      for (const dependency of coValue.missingDependencies) {
+        const dependencyCoValue = this.local.getCoValue(dependency);
+        if (!dependencyCoValue.hasVerifiedContent()) {
+          const peers = this.getServerPeers(dependency);
+
+          // If the peer that sent the new content is a client, we can assume that they are in possession of the dependency
+          if (peer?.role === "client") {
+            peers.push(peer);
+          }
+
+          dependencyCoValue.load(peers);
+        }
+      }
+
+      return;
+    }
+
+    /**
+     * Check if we have the CoValue in memory
+     */
     if (!coValue.hasVerifiedContent()) {
+      /**
+       * The peer has assumed we already have the CoValue
+       */
       if (!msg.header) {
+        // We check if the covalue was in memory and has been garbage collected
+        // In that case we should have it tracked in the storage
         const storageKnownState = this.local.storage?.getKnownState(msg.id);
 
         if (storageKnownState?.header) {
@@ -542,6 +572,7 @@ export class SyncManager {
           return;
         }
 
+        // The peer assumption is not correct, so we ask for the full CoValue
         if (peer) {
           this.trySendToPeer(peer, {
             action: "known",
@@ -551,6 +582,8 @@ export class SyncManager {
             sessions: {},
           });
         } else {
+          // The wrong assumption has been made by storage or import, we don't have a recovery mechanism
+          // Should never happen
           logger.error(
             "Received new content with no header on a missing CoValue",
             {
@@ -561,43 +594,15 @@ export class SyncManager {
         return;
       }
 
-      const sessionIDs = Object.keys(msg.new) as SessionID[];
-      const transactions = Object.values(msg.new).map(
-        (content) => content.newTransactions,
-      );
-
-      // If we'll be performing transaction verification, ensure all the dependencies available.
-      if (!this.skipVerify) {
-        for (const dependency of getDependedOnCoValuesFromRawData(
-          msg.id,
-          msg.header,
-          sessionIDs,
-          transactions,
-        )) {
-          const dependencyCoValue = this.local.getCoValue(dependency);
-
-          if (!dependencyCoValue.hasVerifiedContent()) {
-            coValue.markMissingDependency(dependency);
-
-            const peers = this.getServerPeers(dependencyCoValue.id);
-
-            // if the peer that sent the content is a client, we add it to the list of peers
-            // to also ask them for the dependency
-            if (peer?.role === "client") {
-              peers.push(peer);
-            }
-
-            dependencyCoValue.load(peers);
-          } else if (!dependencyCoValue.isAvailable()) {
-            coValue.markMissingDependency(dependency);
-          }
-        }
-      }
+      const previousState = coValue.loadingState;
 
+      /**
+       * We are getting the full CoValue, so we can instantiate it
+       */
       const success = coValue.provideHeader(
         msg.header,
-        peer?.id ?? "storage",
         msg.expectContentUntil,
+        this.skipVerify,
       );
 
       if (!success) {
@@ -608,6 +613,7 @@ export class SyncManager {
         return;
       }
 
+      coValue.markFoundInPeer(peer?.id ?? "storage", previousState);
       peer?.updateHeader(msg.id, true);
 
       if (msg.expectContentUntil) {
@@ -619,6 +625,7 @@ export class SyncManager {
       }
     }
 
+    // At this point the CoValue must be in memory, if not we have a bug
     if (!coValue.hasVerifiedContent()) {
       throw new Error(
         "Unreachable: CoValue should always have a verified state at this point",
@@ -635,6 +642,9 @@ export class SyncManager {
       new: {},
     };
 
+    /**
+     * The coValue is in memory, load the transactions from the content message
+     */
     for (const [sessionID, newContentForSession] of Object.entries(msg.new) as [
       SessionID,
       SessionNewContent,
@@ -659,57 +669,8 @@ export class SyncManager {
         continue;
       }
 
-      // If we'll be performing transaction verification, ensure the account is available.
-      if (!this.skipVerify) {
-        const accountId = accountOrAgentIDfromSessionID(sessionID);
-
-        if (isAccountID(accountId)) {
-          const account = this.local.getCoValue(accountId);
-
-          // We can't verify the transaction without the account, so we delay the session content handling until the account is available
-          if (!account.isAvailable()) {
-            // This covers the case where we are getting a new session on an already loaded coValue
-            // where we need to load the account to get their public key
-            if (!coValue.missingDependencies.has(accountId)) {
-              const peers = this.getServerPeers(account.id);
-
-              if (peer?.role === "client") {
-                // if the peer that sent the content is a client, we add it to the list of peers
-                // to also ask them for the dependency
-                peers.push(peer);
-              }
-
-              account.load(peers);
-            }
-
-            // We need to wait for the account to be available before we can verify the transaction
-            // Currently doing this by delaying the handleNewContent for the session to when we have the account
-            //
-            // This is not the best solution, because the knownState is not updated and the ACK response will be given
-            // by excluding the session.
-            // This is good enough implementation for now because the only case for the account to be missing are out-of-order
-            // dependencies push, so the gap should be short lived.
-            //
-            // When we are going to have sharded-peers we should revisit this, and store unverified sessions that are considered as part of the
-            // knwonState, but not actively used until they can be verified.
-            void account.waitForAvailable().then(() => {
-              this.handleNewContent(
-                {
-                  action: "content",
-                  id: coValue.id,
-                  new: {
-                    [sessionID]: newContentForSession,
-                  },
-                  priority: msg.priority,
-                },
-                from,
-              );
-            });
-            continue;
-          }
-        }
-      }
-
+      // TODO: Handle invalid signatures in the middle of streaming
+      // This could cause a situation where we are unable to load a chunk, and ask for a correction for all the subsequent chunks
       const result = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
@@ -725,6 +686,7 @@ export class SyncManager {
             id: msg.id,
             err: result.error,
           });
+          // TODO Mark only the session as errored, not the whole coValue
           coValue.markErrored(peer.id, result.error);
         } else {
           logger.error("Failed to add transactions from storage", {
@@ -740,7 +702,10 @@ export class SyncManager {
       }
 
       // The new content for this session has been verified, so we can store it
-      contentToStore.new[sessionID] = newContentForSession;
+      if (result.value) {
+        contentToStore.new[sessionID] = newContentForSession;
+      }
+
       peer?.updateSessionCounter(
         msg.id,
         sessionID,
@@ -749,6 +714,9 @@ export class SyncManager {
       );
     }
 
+    /**
+     * Check if we lack some transactions to be able to load the new content
+     */
     if (invalidStateAssumed) {
       if (peer) {
         this.trySendToPeer(peer, {
@@ -784,6 +752,9 @@ export class SyncManager {
 
     const syncedPeers = [];
 
+    /**
+     * Store the content and propagate it to the server peers and the subscribed client peers
+     */
     const hasNewContent =
       contentToStore.header || Object.keys(contentToStore.new).length > 0;
 
@@ -824,6 +795,9 @@ export class SyncManager {
       }
     }
 
+    /**
+     * Send an update to all the sync state listeners
+     */
     for (const peer of syncedPeers) {
       this.syncState.triggerUpdate(peer.id, coValue.id);
     }

package/src/tests/GarbageCollector.test.ts

@@ -69,6 +69,26 @@ describe("garbage collector", () => {
   test("coValues are not garbage collected if they are a group or account", async () => {
     const client = await setupTestAccount();
 
+    client.addStorage({
+      ourName: "client",
+    });
+    client.node.enableGarbageCollector({
+      garbageCollectGroups: true,
+    });
+
+    const group = client.node.createGroup();
+
+    await new Promise((resolve) => setTimeout(resolve, 10));
+
+    client.node.garbageCollector?.collect();
+
+    expect(client.node.getCoValue(group.id).isAvailable()).toBe(false);
+    expect(client.node.getCoValue(client.accountID).isAvailable()).toBe(false);
+  });
+
+  test("group or account coValues are garbage collected if garbageCollectGroups is true", async () => {
+    const client = await setupTestAccount();
+
     client.addStorage({
       ourName: "client",
     });

package/src/tests/SyncStateManager.test.ts

@@ -333,9 +333,9 @@ describe("SyncStateManager", () => {
       [
         "server -> client | CONTENT Map header: true new: After: 0 New: 1",
         "client -> server | LOAD Group sessions: empty",
-        "client -> server | KNOWN Map sessions: header/1",
         "server -> client | CONTENT Group header: true new: After: 0 New: 3",
         "client -> server | KNOWN Group sessions: header/3",
+        "client -> server | KNOWN Map sessions: header/1",
       ]
     `);
   });

package/src/tests/coValueCore.dependencies.test.ts

@@ -0,0 +1,90 @@
+import { beforeEach, expect, test } from "vitest";
+import {
+  loadCoValueOrFail,
+  setupTestAccount,
+  setupTestNode,
+} from "./testUtils.js";
+
+beforeEach(() => {
+  setupTestNode({ isSyncServer: true });
+});
+
+test("should track the group dependency when creating a new coValue", () => {
+  const client = setupTestNode();
+
+  const group = client.node.createGroup();
+  const map = group.createMap();
+
+  expect(map.core.getDependedOnCoValues()).toEqual(new Set([group.core.id]));
+});
+
+test("should track the source dependency when creating a new branch", () => {
+  const client = setupTestNode();
+
+  const group = client.node.createGroup();
+  const map = group.createMap();
+
+  const branch = map.core.createBranch("feature-branch", group.id);
+
+  expect(branch.getDependedOnCoValues()).toEqual(
+    new Set([map.core.id, group.core.id]),
+  );
+});
+
+test("should track the parent group dependency when extending a group", () => {
+  const client = setupTestNode();
+
+  const group = client.node.createGroup();
+  const parentGroup = client.node.createGroup();
+
+  group.extend(parentGroup);
+
+  expect(group.core.getDependedOnCoValues()).toEqual(
+    new Set([parentGroup.core.id]),
+  );
+});
+
+test("should track the account dependency when syncing an account session", async () => {
+  const sourceClient = await setupTestAccount({
+    connected: true,
+  });
+  const targetClient = await setupTestAccount({
+    connected: true,
+  });
+
+  const group = sourceClient.node.createGroup();
+  group.addMember("everyone", "reader");
+  const map = group.createMap();
+
+  map.set("hello", "world");
+
+  await map.core.waitForSync();
+
+  const loadedMap = await loadCoValueOrFail(targetClient.node, map.id);
+
+  expect(loadedMap.core.getDependedOnCoValues()).toEqual(
+    new Set([group.core.id, sourceClient.accountID]),
+  );
+});
+
+test("should track the account dependency when syncing a group extension", async () => {
+  const sourceClient = await setupTestAccount({
+    connected: true,
+  });
+  const targetClient = await setupTestAccount({
+    connected: true,
+  });
+
+  const group = sourceClient.node.createGroup();
+  const parentGroup = sourceClient.node.createGroup();
+
+  group.extend(parentGroup);
+
+  await group.core.waitForSync();
+
+  const loadedGroup = await loadCoValueOrFail(targetClient.node, group.id);
+
+  expect(loadedGroup.core.getDependedOnCoValues()).toEqual(
+    new Set([parentGroup.core.id, sourceClient.accountID]),
+  );
+});

package/src/tests/coValueCore.test.ts

@@ -539,7 +539,7 @@ describe("markErrored and isErroredInPeer", () => {
 
     expect(coValue.isAvailable()).toBe(false);
 
-    const success = coValue.provideHeader(header, "peerId");
+    const success = coValue.provideHeader(header);
     expect(success).toBe(true);
     expect(coValue.isAvailable()).toBe(true);
   });
@@ -559,7 +559,7 @@ describe("markErrored and isErroredInPeer", () => {
 
     expect(coValue.isAvailable()).toBe(false);
 
-    const success = coValue.provideHeader(header, "peerId");
+    const success = coValue.provideHeader(header);
     expect(success).toBe(false);
     expect(coValue.isAvailable()).toBe(false);
   });