cojson 0.13.27 → 0.13.28

package/src/sync.ts

@@ -11,6 +11,8 @@ import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
 import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
+import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
+import { isAccountID } from "./typeUtils/isAccountID.js";
 
 export type CoValueKnownState = {
   id: RawCoID;
@@ -211,9 +213,9 @@ export class SyncManager {
       return;
     }
 
-    coValue
-      .getDependedOnCoValues()
-      .map((id) => this.sendNewContentIncludingDependencies(id, peer));
+    for (const dependency of coValue.getDependedOnCoValues()) {
+      this.sendNewContentIncludingDependencies(dependency, peer);
+    }
 
     const newContentPieces = coValue.verified.newContentSince(
       peer.optimisticKnownStates.get(id),
@@ -298,7 +300,7 @@ export class SyncManager {
     }
   }
 
-  async addPeer(peer: Peer) {
+  addPeer(peer: Peer) {
     const prevPeer = this.peers[peer.id];
 
     if (prevPeer && !prevPeer.closed) {
@@ -379,65 +381,17 @@ export class SyncManager {
     peer.setKnownState(msg.id, knownStateIn(msg));
     const coValue = this.local.getCoValue(msg.id);
 
-    if (
-      coValue.loadingState === "unknown" ||
-      coValue.loadingState === "unavailable"
-    ) {
-      const eligiblePeers = this.getServerAndStoragePeers(peer.id);
-
-      if (eligiblePeers.length === 0) {
-        // We don't have any eligible peers to load the coValue from
-        // so we send a known state back to the sender to let it know
-        // that the coValue is unavailable
-        peer.trackToldKnownState(msg.id);
-        this.trySendToPeer(peer, {
-          action: "known",
-          id: msg.id,
-          header: false,
-          sessions: {},
-        });
-
-        return;
-      } else {
-        // Syncronously updates the state loading is possible
-        coValue
-          .loadFromPeers(this.getServerAndStoragePeers(peer.id))
-          .catch((e) => {
-            logger.error("Error loading coValue in handleLoad", { err: e });
-          });
-      }
+    if (coValue.isAvailable()) {
+      this.sendNewContentIncludingDependencies(msg.id, peer);
+      return;
     }
 
-    if (coValue.loadingState === "loading") {
-      // We need to return from handleLoad immediately and wait for the CoValue to be loaded
-      // in a new task, otherwise we might block further incoming content messages that would
-      // resolve the CoValue as available. This can happen when we receive fresh
-      // content from a client, but we are a server with our own upstream server(s)
-      coValue
-        .waitForAvailableOrUnavailable()
-        .then(async (value) => {
-          if (!value.isAvailable()) {
-            peer.trackToldKnownState(msg.id);
-            this.trySendToPeer(peer, {
-              action: "known",
-              id: msg.id,
-              header: false,
-              sessions: {},
-            });
-
-            return;
-          }
+    const eligiblePeers = this.getServerAndStoragePeers(peer.id);
 
-          this.sendNewContentIncludingDependencies(msg.id, peer);
-        })
-        .catch((e) => {
-          logger.error("Error loading coValue in handleLoad loading state", {
-            err: e,
-          });
-        });
-    } else if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
-    } else {
+    if (eligiblePeers.length === 0) {
+      // We don't have any eligible peers to load the coValue from
+      // so we send a known state back to the sender to let it know
+      // that the coValue is unavailable
       peer.trackToldKnownState(msg.id);
       this.trySendToPeer(peer, {
         action: "known",
@@ -445,9 +399,41 @@ export class SyncManager {
         header: false,
         sessions: {},
       });
+
+      return;
     }
-  }
 
+    coValue.loadFromPeers(eligiblePeers).catch((e) => {
+      logger.error("Error loading coValue in handleLoad", { err: e });
+    });
+
+    // We need to return from handleLoad immediately and wait for the CoValue to be loaded
+    // in a new task, otherwise we might block further incoming content messages that would
+    // resolve the CoValue as available. This can happen when we receive fresh
+    // content from a client, but we are a server with our own upstream server(s)
+    coValue
+      .waitForAvailableOrUnavailable()
+      .then((value) => {
+        if (!value.isAvailable()) {
+          peer.trackToldKnownState(msg.id);
+          this.trySendToPeer(peer, {
+            action: "known",
+            id: msg.id,
+            header: false,
+            sessions: {},
+          });
+
+          return;
+        }
+
+        this.sendNewContentIncludingDependencies(msg.id, peer);
+      })
+      .catch((e) => {
+        logger.error("Error loading coValue in handleLoad loading state", {
+          err: e,
+        });
+      });
+  }
   handleKnownState(msg: KnownStateMessage, peer: PeerState) {
     const coValue = this.local.getCoValue(msg.id);
 
@@ -494,6 +480,52 @@ export class SyncManager {
         return;
       }
 
+      let dependencyMissing = false;
+      const sessionIDs = Object.keys(msg.new) as SessionID[];
+      for (const dependency of coValue.getDependedOnCoValuesFromHeaderAndSessions(
+        msg.header,
+        sessionIDs,
+      )) {
+        const dependencyCoValue = this.local.getCoValue(dependency);
+
+        if (!dependencyCoValue.isAvailable()) {
+          if (peer.role !== "storage") {
+            this.trySendToPeer(peer, {
+              action: "load",
+              id: dependency,
+              header: false,
+              sessions: {},
+            });
+          }
+
+          dependencyMissing = true;
+        }
+      }
+
+      if (dependencyMissing) {
+        if (peer.role !== "storage") {
+          /**
+           * If we have missing dependencies, we send a known state message to the peer
+           * to let it know that we need a correction update.
+           *
+           * Sync-wise is sub-optimal, but it gives us correctness until
+           * https://github.com/garden-co/jazz/issues/1917 is implemented.
+           */
+          this.trySendToPeer(peer, {
+            action: "known",
+            isCorrection: true,
+            id: msg.id,
+            header: false,
+            sessions: {},
+          });
+        } else {
+          /** Cases of broken deps from storage are recovered by falling back to the server peers */
+          coValue.loadFromPeers(this.getServerAndStoragePeers(peer.id));
+        }
+
+        return;
+      }
+
       peer.updateHeader(msg.id, true);
       coValue.markAvailable(msg.header, peer.id);
     }
@@ -528,6 +560,18 @@ export class SyncManager {
         continue;
       }
 
+      const accountId = accountOrAgentIDfromSessionID(sessionID);
+
+      if (isAccountID(accountId)) {
+        const account = this.local.getCoValue(accountId);
+
+        if (!account.isAvailable()) {
+          account.loadFromPeers([peer]);
+          invalidStateAssumed = true;
+          continue;
+        }
+      }
+
       const result = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
@@ -607,6 +651,8 @@ export class SyncManager {
         // Check if there is a inflight load operation and we
         // are waiting for other peers to send the load request
         if (state === "unknown" || state === undefined) {
+          // Sending a load message to the peer to get to know how much content is missing
+          // before sending the new content
           this.trySendToPeer(peer, {
             action: "load",
             ...coValue.knownState(),
@@ -645,7 +691,7 @@ export class SyncManager {
     this.requestedSyncs.add(coValue.id);
   }
 
-  async syncCoValue(coValue: CoValueCore) {
+  syncCoValue(coValue: CoValueCore) {
     this.requestedSyncs.delete(coValue.id);
 
     for (const peer of this.peersInPriorityOrder()) {
@@ -668,21 +714,21 @@ export class SyncManager {
     }
   }
 
-  async waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
+  waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const { syncState } = this;
     const currentSyncState = syncState.getCurrentSyncState(peerId, id);
 
     const isTheConditionAlreadyMet = currentSyncState.uploaded;
 
     if (isTheConditionAlreadyMet) {
-      return true;
+      return;
     }
 
     const peerState = this.peers[peerId];
 
     // The peer has been closed, so it isn't possible to sync
     if (!peerState || peerState.closed) {
-      return true;
+      return;
     }
 
     // The client isn't subscribed to the coValue, so we won't sync it
@@ -690,7 +736,7 @@ export class SyncManager {
       peerState.role === "client" &&
       !peerState.optimisticKnownStates.has(id)
     ) {
-      return true;
+      return;
     }
 
     return new Promise((resolve, reject) => {
@@ -712,25 +758,25 @@ export class SyncManager {
     });
   }
 
-  async waitForStorageSync(id: RawCoID, timeout = 30_000) {
+  waitForStorageSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    await Promise.all(
+    return Promise.all(
       peers
         .filter((peer) => peer.role === "storage")
         .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }
 
-  async waitForSync(id: RawCoID, timeout = 30_000) {
+  waitForSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    await Promise.all(
+    return Promise.all(
       peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }
 
-  async waitForAllCoValuesSync(timeout = 60_000) {
+  waitForAllCoValuesSync(timeout = 60_000) {
     const coValues = this.local.allCoValues();
     const validCoValues = Array.from(coValues).filter(
       (coValue) =>

package/src/tests/SyncStateManager.test.ts

@@ -257,7 +257,7 @@ describe("SyncStateManager", () => {
     const group = client.node.createGroup();
     const map = group.createMap();
 
-    await expect(map.core.waitForSync()).resolves.toBeUndefined();
+    await map.core.waitForSync();
   });
 
   test("should skip client peers that are not subscribed to the coValue", async () => {

package/src/tests/sync.mesh.test.ts

@@ -4,6 +4,7 @@ import { expectMap } from "../coValue";
 import {
   SyncMessagesLog,
   blockMessageTypeOnOutgoingPeer,
+  connectedPeersWithMessagesTracking,
   loadCoValueOrFail,
   setupTestNode,
   waitFor,
@@ -76,8 +77,8 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
         "core -> storage | CONTENT Group header: true new: After: 0 New: 3",
         "storage -> core | KNOWN Group sessions: header/3",
         "core -> storage | CONTENT Map header: true new: After: 0 New: 1",
-        "storage -> core | KNOWN Map sessions: header/1",
         "client -> edge-italy | LOAD Map sessions: empty",
+        "storage -> core | KNOWN Map sessions: header/1",
         "edge-italy -> core | LOAD Map sessions: empty",
         "core -> edge-italy | CONTENT Group header: true new: After: 0 New: 3",
         "edge-italy -> core | KNOWN Group sessions: header/3",
@@ -138,8 +139,8 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
         "core -> storage | CONTENT Group header: true new: After: 0 New: 5",
         "storage -> core | KNOWN Group sessions: header/5",
         "core -> storage | CONTENT Map header: true new: After: 0 New: 1",
-        "storage -> core | KNOWN Map sessions: header/1",
         "client -> edge-italy | LOAD Map sessions: empty",
+        "storage -> core | KNOWN Map sessions: header/1",
         "edge-italy -> core | LOAD Map sessions: empty",
         "core -> edge-italy | CONTENT ParentGroup header: true new: After: 0 New: 6",
         "edge-italy -> core | KNOWN ParentGroup sessions: header/6",
@@ -355,6 +356,7 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
       syncServer: storage.node,
     });
 
+    storagePeer.role = "storage";
     storagePeer.priority = 100;
 
     const group = coreServer.node.createGroup();
@@ -402,4 +404,60 @@ describe("multiple clients syncing with the a cloud-like server mesh", () => {
 
     expect(mapOnClient.get("hello")).toEqual("world");
   });
+
+  test("a stuck server peer should not block the load from other server peers", async () => {
+    const client = setupTestNode();
+    const coreServer = setupTestNode({
+      isSyncServer: true,
+    });
+
+    const anotherServer = setupTestNode({});
+
+    const { peer: peerToCoreServer } = client.connectToSyncServer({
+      syncServerName: "core",
+      syncServer: coreServer.node,
+    });
+
+    const { peer1, peer2 } = connectedPeersWithMessagesTracking({
+      peer1: {
+        id: anotherServer.node.getCurrentAgent().id,
+        role: "server",
+        name: "another-server",
+      },
+      peer2: {
+        id: client.node.getCurrentAgent().id,
+        role: "client",
+        name: "client",
+      },
+    });
+
+    blockMessageTypeOnOutgoingPeer(peerToCoreServer, "load");
+
+    client.node.syncManager.addPeer(peer1);
+    anotherServer.node.syncManager.addPeer(peer2);
+
+    const group = anotherServer.node.createGroup();
+    const map = group.createMap();
+
+    map.set("hello", "world", "trusting");
+
+    const mapOnClient = await loadCoValueOrFail(client.node, map.id);
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> another-server | LOAD Map sessions: empty",
+        "another-server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "client -> another-server | KNOWN Group sessions: header/3",
+        "another-server -> client | CONTENT Map header: true new: After: 0 New: 1",
+        "client -> another-server | KNOWN Map sessions: header/1",
+      ]
+    `);
+
+    expect(mapOnClient.get("hello")).toEqual("world");
+  });
 });

package/src/tests/sync.peerReconciliation.test.ts

@@ -1,7 +1,13 @@
 import { assert, beforeEach, describe, expect, test } from "vitest";
 import { expectMap } from "../coValue";
 import { WasmCrypto } from "../crypto/WasmCrypto";
-import { SyncMessagesLog, setupTestNode, waitFor } from "./testUtils";
+import {
+  SyncMessagesLog,
+  loadCoValueOrFail,
+  setupTestAccount,
+  setupTestNode,
+  waitFor,
+} from "./testUtils";
 
 let jazzCloud = setupTestNode({ isSyncServer: true });
 
@@ -146,7 +152,9 @@ describe("peer reconciliation", () => {
   });
 
   test("correctly handle server restarts in the middle of a sync", async () => {
-    const client = setupTestNode();
+    const client = setupTestNode({
+      connected: true,
+    });
 
     const group = client.node.createGroup();
     const map = group.createMap();
@@ -182,10 +190,90 @@ describe("peer reconciliation", () => {
         "server -> client | KNOWN Group sessions: empty",
         "client -> server | LOAD Map sessions: header/2",
         "server -> client | KNOWN Map sessions: empty",
+        "client -> server | CONTENT Map header: false new: After: 1 New: 1",
+        "server -> client | KNOWN CORRECTION Map sessions: empty",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | LOAD Group sessions: empty",
         "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN CORRECTION Map sessions: empty",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | KNOWN Group sessions: header/3",
+        "server -> client | KNOWN Map sessions: header/2",
+        "client -> server | LOAD Group sessions: header/3",
         "server -> client | KNOWN Group sessions: header/3",
+        "client -> server | LOAD Map sessions: header/2",
+        "server -> client | KNOWN Map sessions: header/2",
+      ]
+    `);
+  });
+
+  test("correctly handle server restarts in the middle of a sync (2 - account)", async () => {
+    const client = await setupTestAccount({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.set("hello", "world", "trusting");
+
+    await map.core.waitForSync();
+
+    jazzCloud.restart();
+    SyncMessagesLog.clear();
+    client.connectToSyncServer();
+
+    map.set("hello", "updated", "trusting");
+
+    await new Promise((resolve) => setTimeout(resolve, 0));
+
+    client.connectToSyncServer();
+
+    await waitFor(() => {
+      const mapOnSyncServer = jazzCloud.node.getCoValue(map.id);
+
+      expect(mapOnSyncServer.loadingState).toBe("available");
+    });
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Account: client.node.expectCurrentAccount("client account").core,
+        Profile: client.node.expectProfileLoaded(client.accountID).core,
+        ProfileGroup: client.node.expectProfileLoaded(client.accountID).group
+          .core,
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | LOAD Account sessions: header/4",
+        "server -> client | KNOWN Account sessions: empty",
+        "client -> server | LOAD ProfileGroup sessions: header/5",
+        "server -> client | KNOWN ProfileGroup sessions: empty",
+        "client -> server | LOAD Profile sessions: header/1",
+        "server -> client | KNOWN Profile sessions: empty",
+        "client -> server | LOAD Group sessions: header/3",
+        "server -> client | KNOWN Group sessions: empty",
+        "client -> server | LOAD Map sessions: header/2",
+        "server -> client | KNOWN Map sessions: empty",
+        "client -> server | CONTENT Map header: false new: After: 1 New: 1",
+        "server -> client | KNOWN CORRECTION Map sessions: empty",
         "client -> server | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | LOAD Account sessions: empty",
+        "client -> server | CONTENT Account header: true new: After: 0 New: 4",
+        "server -> client | LOAD Group sessions: empty",
+        "client -> server | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | KNOWN CORRECTION Map sessions: empty",
+        "client -> server | CONTENT Map header: true new: After: 0 New: 2",
+        "server -> client | KNOWN Account sessions: header/4",
+        "server -> client | KNOWN Group sessions: header/3",
         "server -> client | KNOWN Map sessions: header/2",
+        "client -> server | LOAD Account sessions: header/4",
+        "server -> client | KNOWN Account sessions: header/4",
+        "client -> server | LOAD ProfileGroup sessions: header/5",
+        "server -> client | KNOWN ProfileGroup sessions: empty",
+        "client -> server | LOAD Profile sessions: header/1",
+        "server -> client | KNOWN Profile sessions: empty",
         "client -> server | LOAD Group sessions: header/3",
         "server -> client | KNOWN Group sessions: header/3",
         "client -> server | LOAD Map sessions: header/2",

package/src/tests/testUtils.ts

@@ -463,6 +463,7 @@ export function createMockStoragePeer(opts: {
     },
   });
 
+  peer1.role = "storage";
   peer1.priority = 100;
 
   storage.syncManager.addPeer(peer2);