cojson 0.13.5 → 0.13.7

package/src/permissions.ts

@@ -225,6 +225,7 @@ function determineValidTransactionsForGroup(
   });
 
   const memberState: MemberState = {};
+  const writeOnlyKeys: Record<RawAccountID | AgentID, KeyID> = {};
   const validTransactions: ValidTransactionsResult[] = [];
 
   const keyRevelations = new Set<string>();
@@ -302,7 +303,8 @@ function determineValidTransactionsForGroup(
         memberState[transactor] !== "adminInvite" &&
         memberState[transactor] !== "writerInvite" &&
         memberState[transactor] !== "readerInvite" &&
-        memberState[transactor] !== "writeOnlyInvite"
+        memberState[transactor] !== "writeOnlyInvite" &&
+        !isOwnWriteKeyRevelation(change.key, transactor, writeOnlyKeys)
       ) {
         logPermissionError("Only admins can reveal keys");
         continue;
@@ -370,14 +372,19 @@ function determineValidTransactionsForGroup(
       validTransactions.push({ txID: { sessionID, txIndex }, tx });
       continue;
     } else if (isWriteKeyForMember(change.key)) {
+      const memberKey = getAccountOrAgentFromWriteKeyForMember(change.key);
+
       if (
         memberState[transactor] !== "admin" &&
-        memberState[transactor] !== "writeOnlyInvite"
+        memberState[transactor] !== "writeOnlyInvite" &&
+        memberKey !== transactor
       ) {
         logPermissionError("Only admins can set writeKeys");
         continue;
       }
 
+      writeOnlyKeys[memberKey] = change.value as KeyID;
+
       /**
        * writeOnlyInvite need to be able to set writeKeys because every new writeOnly
        * member comes with their own write key.
@@ -422,11 +429,12 @@ function determineValidTransactionsForGroup(
       !(
         change.value === "reader" ||
         change.value === "writer" ||
+        change.value === "writeOnly" ||
         change.value === "revoked"
       )
     ) {
       logPermissionError(
-        "Everyone can only be set to reader, writer or revoked",
+        "Everyone can only be set to reader, writer, writeOnly or revoked",
       );
       continue;
     }
@@ -499,6 +507,12 @@ export function isWriteKeyForMember(
   return co.startsWith("writeKeyFor_");
 }
 
+export function getAccountOrAgentFromWriteKeyForMember(
+  co: `writeKeyFor_${RawAccountID | AgentID}`,
+): RawAccountID | AgentID {
+  return co.slice("writeKeyFor_".length) as RawAccountID | AgentID;
+}
+
 export function isKeyForKeyField(co: string): co is `${KeyID}_for_${KeyID}` {
   return co.startsWith("key_") && co.includes("_for_key");
 }
@@ -520,3 +534,17 @@ function isParentExtension(key: string): key is `parent_${CoID<RawGroup>}` {
 function isChildExtension(key: string): key is `child_${CoID<RawGroup>}` {
   return key.startsWith("child_");
 }
+
+function isOwnWriteKeyRevelation(
+  key: `${KeyID}_for_${string}`,
+  memberKey: RawAccountID | AgentID,
+  writeOnlyKeys: Record<RawAccountID | AgentID, KeyID>,
+): key is `${KeyID}_for_${RawAccountID | AgentID}` {
+  if (Object.keys(writeOnlyKeys).length === 0) {
+    return false;
+  }
+
+  const keyID = key.slice(0, key.indexOf("_for_"));
+
+  return writeOnlyKeys[memberKey] === keyID;
+}

package/src/sync.ts

@@ -1,8 +1,9 @@
-import { ValueType, metrics } from "@opentelemetry/api";
+import { Histogram, ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
+import { CoValueState } from "./coValueState.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
@@ -35,8 +36,8 @@ export type LoadMessage = {
 
 export type KnownStateMessage = {
   action: "known";
-  asDependencyOf?: RawCoID;
   isCorrection?: boolean;
+  asDependencyOf?: RawCoID;
 } & CoValueKnownState;
 
 export type NewContentMessage = {
@@ -122,10 +123,19 @@ export class SyncManager {
     valueType: ValueType.INT,
     unit: "peer",
   });
+  private transactionsSizeHistogram: Histogram;
 
   constructor(local: LocalNode) {
     this.local = local;
     this.syncState = new SyncStateManager(this);
+
+    this.transactionsSizeHistogram = metrics
+      .getMeter("cojson")
+      .createHistogram("jazz.transactions.size", {
+        description: "The size of transactions in a covalue",
+        unit: "bytes",
+        valueType: ValueType.INT,
+      });
   }
 
   syncState: SyncStateManager;
@@ -145,7 +155,10 @@ export class SyncManager {
 
   getServerAndStoragePeers(excludePeerId?: PeerID): PeerState[] {
     return this.peersInPriorityOrder().filter(
-      (peer) => peer.isServerOrStoragePeer() && peer.id !== excludePeerId,
+      (peer) =>
+        peer.isServerOrStoragePeer() &&
+        peer.id !== excludePeerId &&
+        !peer.closed,
     );
   }
 
@@ -189,56 +202,35 @@ export class SyncManager {
     }
   }
 
-  async subscribeToIncludingDependencies(id: RawCoID, peer: PeerState) {
-    const entry = this.local.coValuesStore.get(id);
-
-    if (entry.state.type !== "available") {
-      entry.loadFromPeers([peer]).catch((e: unknown) => {
-        logger.error("Error sending load", { err: e });
-      });
-      return;
-    }
-
-    const coValue = entry.state.coValue;
-
-    for (const id of coValue.getDependedOnCoValues()) {
-      await this.subscribeToIncludingDependencies(id, peer);
-    }
-
-    if (!peer.toldKnownState.has(id)) {
-      peer.toldKnownState.add(id);
-      this.trySendToPeer(peer, {
-        action: "load",
-        ...coValue.knownState(),
-      }).catch((e: unknown) => {
-        logger.error("Error sending load", { err: e });
-      });
-    }
-  }
-
-  async tellUntoldKnownStateIncludingDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    asDependencyOf?: RawCoID,
-  ) {
+  async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
     const coValue = this.local.expectCoValueLoaded(id);
 
     await Promise.all(
       coValue
         .getDependedOnCoValues()
-        .map((dependentCoID) =>
-          this.tellUntoldKnownStateIncludingDependencies(
-            dependentCoID,
-            peer,
-            asDependencyOf || id,
-          ),
-        ),
+        .map((id) => this.sendNewContentIncludingDependencies(id, peer)),
+    );
+
+    const newContentPieces = coValue.newContentSince(
+      peer.optimisticKnownStates.get(id),
     );
 
-    if (!peer.toldKnownState.has(id)) {
+    if (newContentPieces) {
+      for (const piece of newContentPieces) {
+        this.trySendToPeer(peer, piece).catch((e: unknown) => {
+          logger.error("Error sending content piece", { err: e });
+        });
+      }
+
+      peer.toldKnownState.add(id);
+      peer.optimisticKnownStates.dispatch({
+        type: "COMBINE_WITH",
+        id: id,
+        value: coValue.knownState(),
+      });
+    } else if (!peer.toldKnownState.has(id)) {
       this.trySendToPeer(peer, {
         action: "known",
-        asDependencyOf,
         ...coValue.knownState(),
       }).catch((e: unknown) => {
         logger.error("Error sending known state", { err: e });
@@ -248,66 +240,101 @@ export class SyncManager {
     }
   }
 
-  async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
-    const coValue = this.local.expectCoValueLoaded(id);
+  async startPeerReconciliation(peer: PeerState) {
+    const coValuesOrderedByDependency: CoValueCore[] = [];
 
-    await Promise.all(
-      coValue
-        .getDependedOnCoValues()
-        .map((id) => this.sendNewContentIncludingDependencies(id, peer)),
-    );
+    const gathered = new Set<string>();
 
-    const newContentPieces = coValue.newContentSince(
-      peer.optimisticKnownStates.get(id),
-    );
+    const buildOrderedCoValueList = (coValue: CoValueCore) => {
+      if (gathered.has(coValue.id)) {
+        return;
+      }
 
-    if (newContentPieces) {
-      const optimisticKnownStateBefore =
-        peer.optimisticKnownStates.get(id) || emptyKnownState(id);
-
-      const sendPieces = async () => {
-        let lastYield = performance.now();
-        for (const [_i, piece] of newContentPieces.entries()) {
-          this.trySendToPeer(peer, piece).catch((e: unknown) => {
-            logger.error("Error sending content piece", { err: e });
-          });
+      gathered.add(coValue.id);
+
+      for (const id of coValue.getDependedOnCoValues()) {
+        const entry = this.local.coValuesStore.get(id);
 
-          if (performance.now() - lastYield > 10) {
-            await new Promise<void>((resolve) => {
-              setTimeout(resolve, 0);
+        if (entry.state.type === "available") {
+          buildOrderedCoValueList(entry.state.coValue);
+        }
+      }
+
+      coValuesOrderedByDependency.push(coValue);
+    };
+
+    for (const entry of this.local.coValuesStore.getValues()) {
+      switch (entry.state.type) {
+        case "unavailable":
+          // If the coValue is unavailable and we never tried this peer
+          // we try to load it from the peer
+          if (!peer.toldKnownState.has(entry.id)) {
+            await entry.loadFromPeers([peer]).catch((e: unknown) => {
+              logger.error("Error sending load", { err: e });
             });
-            lastYield = performance.now();
           }
-        }
-      };
+          break;
+        case "available":
+          const coValue = entry.state.coValue;
+
+          // Build the list of coValues ordered by dependency
+          // so we can send the load message in the correct order
+          buildOrderedCoValueList(coValue);
+          break;
+      }
 
-      sendPieces().catch((e) => {
-        logger.error("Error sending new content piece, retrying", { err: e });
+      // Fill the missing known states with empty known states
+      if (!peer.optimisticKnownStates.has(entry.id)) {
         peer.optimisticKnownStates.dispatch({
-          type: "SET",
-          id,
-          value: optimisticKnownStateBefore ?? emptyKnownState(id),
+          type: "SET_AS_EMPTY",
+          id: entry.id,
         });
-        return this.sendNewContentIncludingDependencies(id, peer);
-      });
+      }
+    }
 
-      peer.optimisticKnownStates.dispatch({
-        type: "COMBINE_WITH",
-        id,
-        value: coValue.knownState(),
+    for (const coValue of coValuesOrderedByDependency) {
+      /**
+       * We send the load messages to:
+       * - Subscribe to the coValue updates
+       * - Start the sync process in case we or the other peer
+       *   lacks some transactions
+       */
+      peer.toldKnownState.add(coValue.id);
+      this.trySendToPeer(peer, {
+        action: "load",
+        ...coValue.knownState(),
+      }).catch((e: unknown) => {
+        logger.error("Error sending load", { err: e });
       });
     }
   }
 
-  addPeer(peer: Peer) {
+  nextPeer: Map<PeerID, Peer> = new Map();
+
+  async addPeer(peer: Peer) {
     const prevPeer = this.peers[peer.id];
-    const peerState = new PeerState(peer, prevPeer?.knownStates);
-    this.peers[peer.id] = peerState;
 
-    if (prevPeer && !prevPeer.closed) {
-      prevPeer.gracefulShutdown();
+    if (prevPeer) {
+      // Assign to nextPeer to check against race conditions
+      prevPeer.nextPeer = peer;
+
+      if (!prevPeer.closed) {
+        prevPeer.gracefulShutdown();
+      }
+
+      // Wait for the previous peer to finish processing the incoming messages
+      await prevPeer.incomingMessagesProcessingPromise?.catch((e) => {});
+
+      // If another peer was added in the meantime, we close this peer
+      if (prevPeer.nextPeer !== peer) {
+        peer.outgoing.close();
+        return;
+      }
     }
 
+    const peerState = new PeerState(peer, prevPeer?.knownStates);
+    this.peers[peer.id] = peerState;
+
     this.peersCounter.add(1, { role: peer.role });
 
     const unsubscribeFromKnownStatesUpdates = peerState.knownStates.subscribe(
@@ -317,43 +344,13 @@ export class SyncManager {
     );
 
     if (peerState.isServerOrStoragePeer()) {
-      const initialSync = async () => {
-        for (const entry of this.local.coValuesStore.getValues()) {
-          await this.subscribeToIncludingDependencies(entry.id, peerState);
-
-          if (entry.state.type === "available") {
-            await this.sendNewContentIncludingDependencies(entry.id, peerState);
-          }
-
-          if (!peerState.optimisticKnownStates.has(entry.id)) {
-            peerState.optimisticKnownStates.dispatch({
-              type: "SET_AS_EMPTY",
-              id: entry.id,
-            });
-          }
-        }
-      };
-      void initialSync();
+      void this.startPeerReconciliation(peerState);
     }
 
-    const processMessages = async () => {
-      for await (const msg of peerState.incoming) {
-        if (msg === "Disconnected") {
-          return;
-        }
-        if (msg === "PingTimeout") {
-          logger.error("Ping timeout from peer", {
-            peerId: peer.id,
-            peerRole: peer.role,
-          });
-          return;
-        }
-
+    peerState
+      .processIncomingMessages(async (msg) => {
         await this.handleSyncMessage(msg, peerState);
-      }
-    };
-
-    processMessages()
+      })
       .then(() => {
         if (peer.crashOnClose) {
           logger.error("Unexepcted close from peer", {
@@ -370,18 +367,18 @@ export class SyncManager {
           peerId: peer.id,
           peerRole: peer.role,
         });
+
         if (peer.crashOnClose) {
           this.local.crashed = e;
           throw new Error(e);
         }
       })
       .finally(() => {
-        const state = this.peers[peer.id];
-        state?.gracefulShutdown();
+        peerState.gracefulShutdown();
         unsubscribeFromKnownStatesUpdates();
         this.peersCounter.add(-1, { role: peer.role });
 
-        if (peer.deletePeerStateOnClose) {
+        if (peer.deletePeerStateOnClose && this.peers[peer.id] === peerState) {
           delete this.peers[peer.id];
         }
       });
@@ -391,7 +388,21 @@ export class SyncManager {
     return peer.pushOutgoingMessage(msg);
   }
 
+  /**
+   * Handles the load message from a peer.
+   *
+   * Differences with the known state message:
+   * - The load message triggers the CoValue loading process on the other peer
+   * - The peer known state is stored as-is instead of being merged
+   * - The load message always replies with a known state message
+   */
   async handleLoad(msg: LoadMessage, peer: PeerState) {
+    /**
+     * We use the msg sessions as source of truth for the known states
+     *
+     * This way we can track part of the data loss that may occur when the other peer is restarted
+     *
+     */
     peer.dispatchToKnownStates({
       type: "SET",
       id: msg.id,
@@ -403,30 +414,23 @@ export class SyncManager {
       const eligiblePeers = this.getServerAndStoragePeers(peer.id);
 
       if (eligiblePeers.length === 0) {
-        // If the load request contains a header or any session data
-        // and we don't have any eligible peers to load the coValue from
-        // we try to load it from the sender because it is the only place
-        // where we can get informations about the coValue
-        if (msg.header || Object.keys(msg.sessions).length > 0) {
-          entry.loadFromPeers([peer]).catch((e) => {
-            logger.error("Error loading coValue in handleLoad", { err: e });
-          });
-        } else {
-          // We don't have any eligible peers to load the coValue from
-          // so we send a known state back to the sender to let it know
-          // that the coValue is unavailable
-          this.trySendToPeer(peer, {
-            action: "known",
-            id: msg.id,
-            header: false,
-            sessions: {},
-          }).catch((e) => {
-            logger.error("Error sending known state back", { err: e });
-          });
-        }
+        // We don't have any eligible peers to load the coValue from
+        // so we send a known state back to the sender to let it know
+        // that the coValue is unavailable
+        peer.toldKnownState.add(msg.id);
+
+        this.trySendToPeer(peer, {
+          action: "known",
+          id: msg.id,
+          header: false,
+          sessions: {},
+        }).catch((e) => {
+          logger.error("Error sending known state back", { err: e });
+        });
 
         return;
       } else {
+        // Should move the state to loading
         this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
           logger.error("Error loading coValue in handleLoad", { err: e });
         });
@@ -442,11 +446,6 @@ export class SyncManager {
         .getCoValue()
         .then(async (value) => {
           if (value === "unavailable") {
-            peer.dispatchToKnownStates({
-              type: "SET",
-              id: msg.id,
-              value: knownStateIn(msg),
-            });
             peer.toldKnownState.add(msg.id);
 
             this.trySendToPeer(peer, {
@@ -461,7 +460,6 @@ export class SyncManager {
             return;
           }
 
-          await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
           await this.sendNewContentIncludingDependencies(msg.id, peer);
         })
         .catch((e) => {
@@ -469,7 +467,9 @@ export class SyncManager {
             err: e,
           });
         });
-    } else if (entry.state.type === "unavailable") {
+    } else if (entry.state.type === "available") {
+      await this.sendNewContentIncludingDependencies(msg.id, peer);
+    } else {
       this.trySendToPeer(peer, {
         action: "known",
         id: msg.id,
@@ -477,11 +477,6 @@ export class SyncManager {
         sessions: {},
       });
     }
-
-    if (entry.state.type === "available") {
-      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-      await this.sendNewContentIncludingDependencies(msg.id, peer);
-    }
   }
 
   async handleKnownState(msg: KnownStateMessage, peer: PeerState) {
@@ -493,31 +488,6 @@ export class SyncManager {
       value: knownStateIn(msg),
     });
 
-    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
-      if (msg.asDependencyOf) {
-        const dependencyEntry = this.local.coValuesStore.get(
-          msg.asDependencyOf,
-        );
-
-        if (
-          dependencyEntry.state.type === "available" ||
-          dependencyEntry.state.type === "loading"
-        ) {
-          this.local
-            .loadCoValueCore(
-              msg.id,
-              peer.role === "storage" ? undefined : peer.id,
-            )
-            .catch((e) => {
-              logger.error(
-                `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
-                { err: e },
-              );
-            });
-        }
-      }
-    }
-
     // The header is a boolean value that tells us if the other peer do have information about the header.
     // If it's false in this point it means that the coValue is unavailable on the other peer.
     if (entry.state.type !== "available") {
@@ -534,11 +504,23 @@ export class SyncManager {
     }
 
     if (entry.state.type === "available") {
-      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
       await this.sendNewContentIncludingDependencies(msg.id, peer);
     }
   }
 
+  recordTransactionsSize(newTransactions: Transaction[], source: string) {
+    for (const tx of newTransactions) {
+      const txLength =
+        tx.privacy === "private"
+          ? tx.encryptedChanges.length
+          : tx.changes.length;
+
+      this.transactionsSizeHistogram.record(txLength, {
+        source,
+      });
+    }
+  }
+
   async handleNewContent(msg: NewContentMessage, peer: PeerState) {
     const entry = this.local.coValuesStore.get(msg.id);
 
@@ -638,6 +620,8 @@ export class SyncManager {
         continue;
       }
 
+      this.recordTransactionsSize(newTransactions, peer.role);
+
       peer.dispatchToKnownStates({
         type: "UPDATE_SESSION_COUNTER",
         id: msg.id,
@@ -660,6 +644,7 @@ export class SyncManager {
           err: e,
         });
       });
+      peer.toldKnownState.add(msg.id);
     } else {
       /**
        * We are sending a known state message to the peer to acknowledge the
@@ -678,6 +663,7 @@ export class SyncManager {
           err: e,
         });
       });
+      peer.toldKnownState.add(msg.id);
     }
 
     /**
@@ -685,7 +671,7 @@ export class SyncManager {
      * response to the peers that are waiting for confirmation that a coValue is
      * fully synced
      */
-    await this.syncCoValue(coValue);
+    this.syncCoValue(coValue);
   }
 
   async handleCorrection(msg: KnownStateMessage, peer: PeerState) {
@@ -729,10 +715,8 @@ export class SyncManager {
       if (peer.erroredCoValues.has(coValue.id)) continue;
 
       if (peer.optimisticKnownStates.has(coValue.id)) {
-        await this.tellUntoldKnownStateIncludingDependencies(coValue.id, peer);
         await this.sendNewContentIncludingDependencies(coValue.id, peer);
       } else if (peer.isServerOrStoragePeer()) {
-        await this.subscribeToIncludingDependencies(coValue.id, peer);
         await this.sendNewContentIncludingDependencies(coValue.id, peer);
       }
     }