cojson 0.13.25 → 0.13.28

package/src/coValueCore/coValueCore.ts

@@ -109,7 +109,7 @@ export class CoValueCore {
   private readonly _decryptionCache: {
     [key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined;
   } = {};
-  private _cachedDependentOn?: RawCoID[];
+  private _cachedDependentOn?: Set<RawCoID>;
   private counter: UpDownCounter;
 
   private constructor(
@@ -897,39 +897,57 @@ export class CoValueCore {
     ];
   }
 
-  getDependedOnCoValues(): RawCoID[] {
+  getDependedOnCoValues(): Set<RawCoID> {
     if (this._cachedDependentOn) {
       return this._cachedDependentOn;
     } else {
-      const dependentOn = this.getDependedOnCoValuesUncached();
+      if (!this.verified) {
+        return new Set();
+      }
+
+      const dependentOn = this.getDependedOnCoValuesFromHeaderAndSessions(
+        this.verified.header,
+        this.verified.sessions.keys(),
+      );
       this._cachedDependentOn = dependentOn;
       return dependentOn;
     }
   }
 
   /** @internal */
-  getDependedOnCoValuesUncached(): RawCoID[] {
-    if (!this.verified) {
-      return [];
+  getDependedOnCoValuesFromHeaderAndSessions(
+    header: CoValueHeader,
+    sessions: Iterable<SessionID>,
+  ): Set<RawCoID> {
+    const deps = new Set<RawCoID>();
+
+    for (const session of sessions) {
+      const accountId = accountOrAgentIDfromSessionID(session);
+
+      if (isAccountID(accountId) && accountId !== this.id) {
+        deps.add(accountId);
+      }
     }
 
-    return this.verified.header.ruleset.type === "group"
-      ? getGroupDependentKeyList(expectGroup(this.getCurrentContent()).keys())
-      : this.verified.header.ruleset.type === "ownedByGroup"
-        ? [
-            this.verified.header.ruleset.group,
-            ...new Set(
-              [...this.verified.sessions.keys()]
-                .map((sessionID) =>
-                  accountOrAgentIDfromSessionID(sessionID as SessionID),
-                )
-                .filter(
-                  (session): session is RawAccountID =>
-                    isAccountID(session) && session !== this.id,
-                ),
-            ),
-          ]
-        : [];
+    if (header.ruleset.type === "group") {
+      if (isAccountID(header.ruleset.initialAdmin)) {
+        deps.add(header.ruleset.initialAdmin);
+      }
+
+      if (this.verified) {
+        for (const id of getGroupDependentKeyList(
+          expectGroup(this.getCurrentContent()).keys(),
+        )) {
+          deps.add(id);
+        }
+      }
+    }
+
+    if (header.ruleset.type === "ownedByGroup") {
+      deps.add(header.ruleset.group);
+    }
+
+    return deps;
   }
 
   waitForSync(options?: {
@@ -943,7 +961,11 @@ export class CoValueCore {
       return;
     }
 
-    const peersToActuallyLoadFrom = [];
+    const peersToActuallyLoadFrom = {
+      storage: [] as PeerState[],
+      server: [] as PeerState[],
+    };
+
     for (const peer of peers) {
       const currentState = this.peers.get(peer.id);
 
@@ -959,78 +981,87 @@ export class CoValueCore {
       }
 
       if (currentState?.type === "unavailable") {
-        if (peer.shouldRetryUnavailableCoValues()) {
+        if (peer.role === "server") {
+          peersToActuallyLoadFrom.server.push(peer);
           this.markPending(peer.id);
-          peersToActuallyLoadFrom.push(peer);
         }
 
         continue;
       }
 
       if (!currentState || currentState?.type === "unknown") {
+        if (peer.role === "storage") {
+          peersToActuallyLoadFrom.storage.push(peer);
+        } else {
+          peersToActuallyLoadFrom.server.push(peer);
+        }
+
         this.markPending(peer.id);
-        peersToActuallyLoadFrom.push(peer);
       }
     }
 
-    for (const peer of peersToActuallyLoadFrom) {
-      if (peer.closed) {
-        this.markNotFoundInPeer(peer.id);
-        continue;
-      }
+    // Load from storage peers first, then from server peers
+    if (peersToActuallyLoadFrom.storage.length > 0) {
+      await Promise.all(
+        peersToActuallyLoadFrom.storage.map((peer) =>
+          this.internalLoadFromPeer(peer),
+        ),
+      );
+    }
 
-      peer.pushOutgoingMessage({
-        action: "load",
-        ...this.knownState(),
-      });
-      peer.trackLoadRequestSent(this.id);
-
-      /**
-       * Use a very long timeout for storage peers, because under pressure
-       * they may take a long time to consume the messages queue
-       *
-       * TODO: Track errors on storage and do not rely on timeout
-       */
-      const timeoutDuration =
-        peer.role === "storage"
-          ? CO_VALUE_LOADING_CONFIG.TIMEOUT * 10
-          : CO_VALUE_LOADING_CONFIG.TIMEOUT;
-
-      const waitingForPeer = new Promise<void>((resolve) => {
-        const markNotFound = () => {
-          if (this.peers.get(peer.id)?.type === "pending") {
-            logger.warn("Timeout waiting for peer to load coValue", {
-              id: this.id,
-              peerID: peer.id,
-            });
-            this.markNotFoundInPeer(peer.id);
-          }
-        };
+    if (peersToActuallyLoadFrom.server.length > 0) {
+      await Promise.all(
+        peersToActuallyLoadFrom.server.map((peer) =>
+          this.internalLoadFromPeer(peer),
+        ),
+      );
+    }
+  }
 
-        const timeout = setTimeout(markNotFound, timeoutDuration);
-        const removeCloseListener = peer.addCloseListener(markNotFound);
+  internalLoadFromPeer(peer: PeerState) {
+    if (peer.closed) {
+      this.markNotFoundInPeer(peer.id);
+      return;
+    }
 
-        const listener = (state: CoValueCore) => {
-          const peerState = state.peers.get(peer.id);
-          if (
-            state.isAvailable() || // might have become available from another peer e.g. through handleNewContent
-            peerState?.type === "available" ||
-            peerState?.type === "errored" ||
-            peerState?.type === "unavailable"
-          ) {
-            this.listeners.delete(listener);
-            removeCloseListener();
-            clearTimeout(timeout);
-            resolve();
-          }
-        };
+    peer.pushOutgoingMessage({
+      action: "load",
+      ...this.knownState(),
+    });
+    peer.trackLoadRequestSent(this.id);
 
-        this.listeners.add(listener);
-        listener(this);
-      });
+    return new Promise<void>((resolve) => {
+      const markNotFound = () => {
+        if (this.peers.get(peer.id)?.type === "pending") {
+          logger.warn("Timeout waiting for peer to load coValue", {
+            id: this.id,
+            peerID: peer.id,
+          });
+          this.markNotFoundInPeer(peer.id);
+        }
+      };
 
-      await waitingForPeer;
-    }
+      const timeout = setTimeout(markNotFound, CO_VALUE_LOADING_CONFIG.TIMEOUT);
+      const removeCloseListener = peer.addCloseListener(markNotFound);
+
+      const listener = (state: CoValueCore) => {
+        const peerState = state.peers.get(peer.id);
+        if (
+          state.isAvailable() || // might have become available from another peer e.g. through handleNewContent
+          peerState?.type === "available" ||
+          peerState?.type === "errored" ||
+          peerState?.type === "unavailable"
+        ) {
+          this.listeners.delete(listener);
+          removeCloseListener();
+          clearTimeout(timeout);
+          resolve();
+        }
+      };
+
+      this.listeners.add(listener);
+      listener(this);
+    });
   }
 }
 

package/src/coValues/group.ts

@@ -773,7 +773,10 @@ export class RawGroup<
   ) {
     const memberKey = typeof account === "string" ? account : account.id;
 
-    this.rotateReadKey(memberKey);
+    if (this.myRole() === "admin") {
+      this.rotateReadKey(memberKey);
+    }
+
     this.set(memberKey, "revoked", "trusting");
   }
 

package/src/localNode.ts

@@ -104,8 +104,12 @@ export class LocalNode {
     this.coValues.delete(id);
   }
 
+  getCurrentAccountOrAgentID(): RawAccountID | AgentID {
+    return accountOrAgentIDfromSessionID(this.currentSessionID);
+  }
+
   getCurrentAgent(): ControlledAccountOrAgent {
-    const accountOrAgent = accountOrAgentIDfromSessionID(this.currentSessionID);
+    const accountOrAgent = this.getCurrentAccountOrAgentID();
     if (isAgentID(accountOrAgent)) {
       return new ControlledAgent(this.agentSecret, this.crypto);
     }
@@ -118,7 +122,7 @@ export class LocalNode {
   }
 
   expectCurrentAccountID(reason: string): RawAccountID {
-    const accountOrAgent = accountOrAgentIDfromSessionID(this.currentSessionID);
+    const accountOrAgent = this.getCurrentAccountOrAgentID();
     if (isAgentID(accountOrAgent)) {
       throw new Error(
         "Current account is an agent, but expected an account: " + reason,

package/src/permissions.ts

@@ -452,7 +452,12 @@ function determineValidTransactionsForGroup(
       change.key === transactor &&
       change.value === "admin";
 
-    if (!isFirstSelfAppointment) {
+    const currentAccountId = coValue.node.getCurrentAccountOrAgentID();
+
+    const isSelfRevoke =
+      currentAccountId === change.key && change.value === "revoked";
+
+    if (!isFirstSelfAppointment && !isSelfRevoke) {
       if (memberState[transactor] === "admin") {
         if (
           memberState[affectedMember] === "admin" &&

package/src/sync.ts

@@ -11,6 +11,8 @@ import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
 import { logger } from "./logger.js";
 import { CoValuePriority } from "./priority.js";
+import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
+import { isAccountID } from "./typeUtils/isAccountID.js";
 
 export type CoValueKnownState = {
   id: RawCoID;
@@ -211,9 +213,9 @@ export class SyncManager {
       return;
     }
 
-    coValue
-      .getDependedOnCoValues()
-      .map((id) => this.sendNewContentIncludingDependencies(id, peer));
+    for (const dependency of coValue.getDependedOnCoValues()) {
+      this.sendNewContentIncludingDependencies(dependency, peer);
+    }
 
     const newContentPieces = coValue.verified.newContentSince(
       peer.optimisticKnownStates.get(id),
@@ -298,7 +300,7 @@ export class SyncManager {
     }
   }
 
-  async addPeer(peer: Peer) {
+  addPeer(peer: Peer) {
     const prevPeer = this.peers[peer.id];
 
     if (prevPeer && !prevPeer.closed) {
@@ -379,65 +381,17 @@ export class SyncManager {
     peer.setKnownState(msg.id, knownStateIn(msg));
     const coValue = this.local.getCoValue(msg.id);
 
-    if (
-      coValue.loadingState === "unknown" ||
-      coValue.loadingState === "unavailable"
-    ) {
-      const eligiblePeers = this.getServerAndStoragePeers(peer.id);
-
-      if (eligiblePeers.length === 0) {
-        // We don't have any eligible peers to load the coValue from
-        // so we send a known state back to the sender to let it know
-        // that the coValue is unavailable
-        peer.trackToldKnownState(msg.id);
-        this.trySendToPeer(peer, {
-          action: "known",
-          id: msg.id,
-          header: false,
-          sessions: {},
-        });
-
-        return;
-      } else {
-        // Syncronously updates the state loading is possible
-        coValue
-          .loadFromPeers(this.getServerAndStoragePeers(peer.id))
-          .catch((e) => {
-            logger.error("Error loading coValue in handleLoad", { err: e });
-          });
-      }
+    if (coValue.isAvailable()) {
+      this.sendNewContentIncludingDependencies(msg.id, peer);
+      return;
     }
 
-    if (coValue.loadingState === "loading") {
-      // We need to return from handleLoad immediately and wait for the CoValue to be loaded
-      // in a new task, otherwise we might block further incoming content messages that would
-      // resolve the CoValue as available. This can happen when we receive fresh
-      // content from a client, but we are a server with our own upstream server(s)
-      coValue
-        .waitForAvailableOrUnavailable()
-        .then(async (value) => {
-          if (!value.isAvailable()) {
-            peer.trackToldKnownState(msg.id);
-            this.trySendToPeer(peer, {
-              action: "known",
-              id: msg.id,
-              header: false,
-              sessions: {},
-            });
-
-            return;
-          }
+    const eligiblePeers = this.getServerAndStoragePeers(peer.id);
 
-          this.sendNewContentIncludingDependencies(msg.id, peer);
-        })
-        .catch((e) => {
-          logger.error("Error loading coValue in handleLoad loading state", {
-            err: e,
-          });
-        });
-    } else if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
-    } else {
+    if (eligiblePeers.length === 0) {
+      // We don't have any eligible peers to load the coValue from
+      // so we send a known state back to the sender to let it know
+      // that the coValue is unavailable
       peer.trackToldKnownState(msg.id);
       this.trySendToPeer(peer, {
         action: "known",
@@ -445,9 +399,41 @@ export class SyncManager {
         header: false,
         sessions: {},
       });
+
+      return;
     }
-  }
 
+    coValue.loadFromPeers(eligiblePeers).catch((e) => {
+      logger.error("Error loading coValue in handleLoad", { err: e });
+    });
+
+    // We need to return from handleLoad immediately and wait for the CoValue to be loaded
+    // in a new task, otherwise we might block further incoming content messages that would
+    // resolve the CoValue as available. This can happen when we receive fresh
+    // content from a client, but we are a server with our own upstream server(s)
+    coValue
+      .waitForAvailableOrUnavailable()
+      .then((value) => {
+        if (!value.isAvailable()) {
+          peer.trackToldKnownState(msg.id);
+          this.trySendToPeer(peer, {
+            action: "known",
+            id: msg.id,
+            header: false,
+            sessions: {},
+          });
+
+          return;
+        }
+
+        this.sendNewContentIncludingDependencies(msg.id, peer);
+      })
+      .catch((e) => {
+        logger.error("Error loading coValue in handleLoad loading state", {
+          err: e,
+        });
+      });
+  }
   handleKnownState(msg: KnownStateMessage, peer: PeerState) {
     const coValue = this.local.getCoValue(msg.id);
 
@@ -494,6 +480,52 @@ export class SyncManager {
         return;
       }
 
+      let dependencyMissing = false;
+      const sessionIDs = Object.keys(msg.new) as SessionID[];
+      for (const dependency of coValue.getDependedOnCoValuesFromHeaderAndSessions(
+        msg.header,
+        sessionIDs,
+      )) {
+        const dependencyCoValue = this.local.getCoValue(dependency);
+
+        if (!dependencyCoValue.isAvailable()) {
+          if (peer.role !== "storage") {
+            this.trySendToPeer(peer, {
+              action: "load",
+              id: dependency,
+              header: false,
+              sessions: {},
+            });
+          }
+
+          dependencyMissing = true;
+        }
+      }
+
+      if (dependencyMissing) {
+        if (peer.role !== "storage") {
+          /**
+           * If we have missing dependencies, we send a known state message to the peer
+           * to let it know that we need a correction update.
+           *
+           * Sync-wise is sub-optimal, but it gives us correctness until
+           * https://github.com/garden-co/jazz/issues/1917 is implemented.
+           */
+          this.trySendToPeer(peer, {
+            action: "known",
+            isCorrection: true,
+            id: msg.id,
+            header: false,
+            sessions: {},
+          });
+        } else {
+          /** Cases of broken deps from storage are recovered by falling back to the server peers */
+          coValue.loadFromPeers(this.getServerAndStoragePeers(peer.id));
+        }
+
+        return;
+      }
+
       peer.updateHeader(msg.id, true);
       coValue.markAvailable(msg.header, peer.id);
     }
@@ -528,6 +560,18 @@ export class SyncManager {
         continue;
       }
 
+      const accountId = accountOrAgentIDfromSessionID(sessionID);
+
+      if (isAccountID(accountId)) {
+        const account = this.local.getCoValue(accountId);
+
+        if (!account.isAvailable()) {
+          account.loadFromPeers([peer]);
+          invalidStateAssumed = true;
+          continue;
+        }
+      }
+
       const result = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
@@ -607,6 +651,8 @@ export class SyncManager {
         // Check if there is a inflight load operation and we
         // are waiting for other peers to send the load request
         if (state === "unknown" || state === undefined) {
+          // Sending a load message to the peer to get to know how much content is missing
+          // before sending the new content
           this.trySendToPeer(peer, {
             action: "load",
             ...coValue.knownState(),
@@ -645,7 +691,7 @@ export class SyncManager {
     this.requestedSyncs.add(coValue.id);
   }
 
-  async syncCoValue(coValue: CoValueCore) {
+  syncCoValue(coValue: CoValueCore) {
     this.requestedSyncs.delete(coValue.id);
 
     for (const peer of this.peersInPriorityOrder()) {
@@ -668,21 +714,21 @@ export class SyncManager {
     }
   }
 
-  async waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
+  waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number) {
     const { syncState } = this;
     const currentSyncState = syncState.getCurrentSyncState(peerId, id);
 
     const isTheConditionAlreadyMet = currentSyncState.uploaded;
 
     if (isTheConditionAlreadyMet) {
-      return true;
+      return;
     }
 
     const peerState = this.peers[peerId];
 
     // The peer has been closed, so it isn't possible to sync
     if (!peerState || peerState.closed) {
-      return true;
+      return;
     }
 
     // The client isn't subscribed to the coValue, so we won't sync it
@@ -690,7 +736,7 @@ export class SyncManager {
       peerState.role === "client" &&
       !peerState.optimisticKnownStates.has(id)
     ) {
-      return true;
+      return;
     }
 
     return new Promise((resolve, reject) => {
@@ -712,25 +758,25 @@ export class SyncManager {
     });
   }
 
-  async waitForStorageSync(id: RawCoID, timeout = 30_000) {
+  waitForStorageSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    await Promise.all(
+    return Promise.all(
       peers
         .filter((peer) => peer.role === "storage")
         .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }
 
-  async waitForSync(id: RawCoID, timeout = 30_000) {
+  waitForSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    await Promise.all(
+    return Promise.all(
       peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }
 
-  async waitForAllCoValuesSync(timeout = 60_000) {
+  waitForAllCoValuesSync(timeout = 60_000) {
     const coValues = this.local.allCoValues();
     const validCoValues = Array.from(coValues).filter(
       (coValue) =>

package/src/tests/SyncStateManager.test.ts

@@ -257,7 +257,7 @@ describe("SyncStateManager", () => {
     const group = client.node.createGroup();
     const map = group.createMap();
 
-    await expect(map.core.waitForSync()).resolves.toBeUndefined();
+    await map.core.waitForSync();
   });
 
   test("should skip client peers that are not subscribed to the coValue", async () => {