cojson 0.18.32 → 0.18.34

package/src/coValueCore/coValueCore.ts

@@ -254,63 +254,77 @@ export class CoValueCore {
     return this.hasVerifiedContent();
   }
 
-  /**
-   * True if the coValue is completely downloaded:
-   * - the current coValue is available and not streaming
-   * - the group is available and not streaming
-   * - TODO: all the parent groups are available and not streaming
-   */
-  isCompletelyDownloaded(): this is AvailableCoValueCore {
+  isCompletelyDownloaded(): boolean {
     if (!this.hasVerifiedContent()) {
       return false;
     }
 
-    if (this.verified.isStreaming()) {
+    if (this.isStreaming()) {
       return false;
     }
 
-    const group = this.safeGetGroup();
-
-    // TODO: Group coValues should be completely downloaded when all their parent groups are completely downloaded
-    if (!group) {
-      return true;
+    if (this.incompleteDependencies.size > 0) {
+      return false;
     }
 
-    return group.core.isCompletelyDownloaded();
+    return true;
+  }
+
+  isStreaming() {
+    return this.verified?.isStreaming() ?? false;
   }
 
   hasVerifiedContent(): this is AvailableCoValueCore {
     return !!this.verified;
   }
 
+  /**
+   * Returns the CoValue data as NewContentMessage objects, excluding the transactions that are part of the given known state.
+   *
+   * Used to serialize the CoValue data to send it to peers and storage.
+   */
+  newContentSince(
+    knownState?: CoValueKnownState,
+  ): NewContentMessage[] | undefined {
+    return this.verified?.newContentSince(knownState);
+  }
+
   isErroredInPeer(peerId: PeerID) {
     return this.getLoadingStateForPeer(peerId) === "errored";
   }
 
-  waitFor(callback: (value: CoValueCore) => boolean) {
+  waitFor(opts: {
+    predicate: (value: CoValueCore) => boolean;
+    onSuccess: (value: CoValueCore) => void;
+  }) {
+    const { predicate, onSuccess } = opts;
+    this.subscribe((core, unsubscribe) => {
+      if (predicate(core)) {
+        unsubscribe();
+        onSuccess(core);
+      }
+    }, true);
+  }
+
+  waitForAsync(callback: (value: CoValueCore) => boolean) {
     return new Promise<CoValueCore>((resolve) => {
-      this.subscribe((core, unsubscribe) => {
-        if (callback(core)) {
-          unsubscribe();
-          resolve(core);
-        }
-      }, true);
+      this.waitFor({ predicate: callback, onSuccess: resolve });
     });
   }
 
   waitForAvailableOrUnavailable(): Promise<CoValueCore> {
-    return this.waitFor(
+    return this.waitForAsync(
       (core) => core.isAvailable() || core.loadingState === "unavailable",
     );
   }
 
   waitForAvailable(): Promise<CoValueCore> {
-    return this.waitFor((core) => core.isAvailable());
+    return this.waitForAsync((core) => core.isAvailable());
   }
 
   waitForFullStreaming(): Promise<CoValueCore> {
-    return this.waitFor(
-      (core) => core.isAvailable() && !core.verified.isStreaming(),
+    return this.waitForAsync(
+      (core) => core.isAvailable() && !core.isStreaming(),
     );
   }
 
@@ -368,7 +382,7 @@ export class CoValueCore {
 
   missingDependencies = new Set<RawCoID>();
 
-  isCircularMissingDependency(dependency: CoValueCore) {
+  isCircularDependency(dependency: CoValueCore) {
     const visited = new Set<RawCoID>();
     const stack = [dependency];
 
@@ -381,7 +395,7 @@ export class CoValueCore {
 
       visited.add(current.id);
 
-      for (const dependency of current.missingDependencies) {
+      for (const dependency of current.dependencies) {
         if (dependency === this.id) {
           return true;
         }
@@ -395,14 +409,6 @@ export class CoValueCore {
     return false;
   }
 
-  markDependencyAvailable(dependency: RawCoID) {
-    this.missingDependencies.delete(dependency);
-
-    if (this.missingDependencies.size === 0) {
-      this.scheduleNotifyUpdate();
-    }
-  }
-
   newContentQueue: {
     msg: NewContentMessage;
     from: PeerState | "storage" | "import";
@@ -422,17 +428,16 @@ export class CoValueCore {
       return;
     }
 
-    this.subscribe((core, unsubscribe) => {
-      if (!core.hasMissingDependencies()) {
-        unsubscribe();
-
+    this.waitFor({
+      predicate: (core) => !core.hasMissingDependencies(),
+      onSuccess: () => {
         const enqueuedNewContent = this.newContentQueue;
         this.newContentQueue = [];
 
         for (const { msg, from } of enqueuedNewContent) {
           this.node.syncManager.handleNewContent(msg, from);
         }
-      }
+      },
     });
   }
 
@@ -764,6 +769,8 @@ export class CoValueCore {
 
     let result: { signature: Signature; transaction: Transaction };
 
+    const knownStateBefore = this.knownState();
+
     if (privacy === "private") {
       const { secret: keySecret, id: keyID } = this.getCurrentReadKey();
 
@@ -790,7 +797,7 @@ export class CoValueCore {
       );
     }
 
-    const { transaction, signature } = result;
+    const { transaction } = result;
 
     // Assign pre-parsed meta and changes to skip the parse/decrypt operation when loading
     // this transaction in the current content
@@ -798,9 +805,6 @@ export class CoValueCore {
 
     this.node.syncManager.recordTransactionsSize([transaction], "local");
 
-    const session = this.verified.sessions.get(sessionID);
-    const txIdx = session ? session.transactions.length - 1 : 0;
-
     this.resetKnownStateCache();
     this.processNewTransactions();
     this.addDependenciesFromNewTransaction(transaction);
@@ -808,13 +812,7 @@ export class CoValueCore {
     // force immediate notification because local updates may come from the UI
     // where we need synchronous updates
     this.notifyUpdate();
-    this.node.syncManager.syncLocalTransaction(
-      this.verified,
-      transaction,
-      sessionID,
-      signature,
-      txIdx,
-    );
+    this.node.syncManager.syncLocalTransaction(this.verified, knownStateBefore);
 
     return true;
   }
@@ -1163,30 +1161,47 @@ export class CoValueCore {
   }
 
   dependencies: Set<RawCoID> = new Set();
+  incompleteDependencies: Set<RawCoID> = new Set();
   private addDependency(dependency: RawCoID) {
-    if (this.dependencies.has(dependency)) {
-      return true;
-    }
-
-    this.dependencies.add(dependency);
-
     const dependencyCoValue = this.node.getCoValue(dependency);
 
-    if (this.isCircularMissingDependency(dependencyCoValue)) {
-      return true;
+    if (
+      this.isCircularDependency(dependencyCoValue) ||
+      this.dependencies.has(dependency)
+    ) {
+      return;
     }
 
+    this.dependencies.add(dependency);
     dependencyCoValue.addDependant(this.id);
 
+    if (!dependencyCoValue.isCompletelyDownloaded()) {
+      this.incompleteDependencies.add(dependencyCoValue.id);
+      dependencyCoValue.waitFor({
+        predicate: (dependencyCoValue) =>
+          dependencyCoValue.isCompletelyDownloaded(),
+        onSuccess: () => {
+          this.incompleteDependencies.delete(dependencyCoValue.id);
+          if (this.incompleteDependencies.size === 0) {
+            // We want this to propagate immediately in the dependency chain
+            this.notifyUpdate();
+          }
+        },
+      });
+    }
+
     if (!dependencyCoValue.isAvailable()) {
-      this.missingDependencies.add(dependency);
-      dependencyCoValue.subscribe((dependencyCoValue, unsubscribe) => {
-        if (dependencyCoValue.isAvailable()) {
-          unsubscribe();
-          this.markDependencyAvailable(dependency);
-        }
+      this.missingDependencies.add(dependencyCoValue.id);
+      dependencyCoValue.waitFor({
+        predicate: (dependencyCoValue) => dependencyCoValue.isAvailable(),
+        onSuccess: () => {
+          this.missingDependencies.delete(dependencyCoValue.id);
+
+          if (this.missingDependencies.size === 0) {
+            this.notifyUpdate(); // We want this to propagate immediately
+          }
+        },
       });
-      return false;
     }
   }
 

package/src/coValueCore/verifiedState.ts

@@ -4,6 +4,8 @@ import {
   createContentMessage,
   exceedsRecommendedSize,
   getTransactionSize,
+  addTransactionToContentMessage,
+  knownStateFromContent,
 } from "../coValueContentMessage.js";
 import {
   CryptoProvider,
@@ -19,9 +21,13 @@ import { JsonObject, JsonValue } from "../jsonValue.js";
 import { PermissionsDef as RulesetDef } from "../permissions.js";
 import { NewContentMessage } from "../sync.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
-import { SessionLog, SessionMap } from "./SessionMap.js";
+import { SessionMap } from "./SessionMap.js";
 import { ControlledAccountOrAgent } from "../coValues/account.js";
-import { cloneKnownState, CoValueKnownState } from "../knownState.js";
+import {
+  CoValueKnownState,
+  getKnownStateToSend,
+  KnownStateSessions,
+} from "../knownState.js";
 
 export type CoValueHeader = {
   type: AnyRawCoValue["type"];
@@ -55,7 +61,6 @@ export class VerifiedState {
   readonly crypto: CryptoProvider;
   readonly header: CoValueHeader;
   readonly sessions: SessionMap;
-  private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   public lastAccessed: number | undefined;
   public branchSourceId?: RawCoID;
   public branchName?: string;
@@ -98,10 +103,6 @@ export class VerifiedState {
       skipVerify,
     );
 
-    if (result.isOk()) {
-      this._cachedNewContentSinceEmpty = undefined;
-    }
-
     return result;
   }
 
@@ -120,8 +121,6 @@ export class VerifiedState {
       madeAt,
     );
 
-    this._cachedNewContentSinceEmpty = undefined;
-
     return result;
   }
 
@@ -144,8 +143,6 @@ export class VerifiedState {
       madeAt,
     );
 
-    this._cachedNewContentSinceEmpty = undefined;
-
     return result;
   }
 
@@ -160,119 +157,157 @@ export class VerifiedState {
     );
   }
 
+  setStreamingKnownState(streamingKnownState: KnownStateSessions) {
+    this.sessions.setStreamingKnownState(streamingKnownState);
+  }
+
   newContentSince(
     knownState: CoValueKnownState | undefined,
+    opts?: {
+      skipExpectContentUntil?: boolean;
+    },
   ): NewContentMessage[] | undefined {
-    const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
-
-    if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
-      return this._cachedNewContentSinceEmpty;
-    }
-
     let currentPiece: NewContentMessage = createContentMessage(
       this.id,
       this.header,
-      !knownState?.header,
+      false,
     );
-
-    const pieces = [currentPiece];
-
-    const sentState: CoValueKnownState["sessions"] = {};
-
+    const pieces: NewContentMessage[] = [currentPiece];
     let pieceSize = 0;
 
-    let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
-
-    while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
-      if (sessionsTodoAgain === "first") {
-        sessionsTodoAgain = undefined;
+    const startNewPiece = () => {
+      currentPiece = createContentMessage(this.id, this.header, false);
+      pieces.push(currentPiece);
+      pieceSize = 0;
+    };
+
+    const moveSessionContentToNewPiece = (sessionID: SessionID) => {
+      const sessionContent = currentPiece.new[sessionID];
+
+      if (!sessionContent) {
+        throw new Error("Session content not found", {
+          cause: {
+            sessionID,
+            currentPiece,
+          },
+        });
       }
-      const sessionsTodo = sessionsTodoAgain ?? this.sessions.keys();
-
-      for (const sessionIDKey of sessionsTodo) {
-        const sessionID = sessionIDKey as SessionID;
-        const log = this.sessions.get(sessionID)!;
-        const knownStateForSessionID = knownState?.sessions[sessionID];
-        const sentStateForSessionID = sentState[sessionID];
-        const nextKnownSignatureIdx = getNextKnownSignatureIdx(
-          log,
-          knownStateForSessionID,
-          sentStateForSessionID,
-        );
-
-        const firstNewTxIdx =
-          sentStateForSessionID ?? knownStateForSessionID ?? 0;
-        const afterLastNewTxIdx =
-          nextKnownSignatureIdx === undefined
-            ? log.transactions.length
-            : nextKnownSignatureIdx + 1;
-
-        const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
-
-        if (nNewTx === 0) {
-          sessionsTodoAgain?.delete(sessionID);
-          continue;
-        }
 
-        if (afterLastNewTxIdx < log.transactions.length) {
-          if (!sessionsTodoAgain) {
-            sessionsTodoAgain = new Set();
+      delete currentPiece.new[sessionID];
+
+      const newPiece = createContentMessage(this.id, this.header, false);
+      newPiece.new[sessionID] = sessionContent;
+
+      // Insert the new piece before the current piece, to ensure that the order of the new transactions is preserved
+      pieces.splice(pieces.length - 1, 0, newPiece);
+    };
+
+    const sessionSent = knownState?.sessions;
+
+    for (const [sessionID, log] of this.sessions.sessions) {
+      const startFrom = sessionSent?.[sessionID] ?? 0;
+
+      let currentSessionSize = 0;
+
+      for (let txIdx = startFrom; txIdx < log.transactions.length; txIdx++) {
+        const isLastItem = txIdx === log.transactions.length - 1;
+        const tx = log.transactions[txIdx]!;
+
+        currentSessionSize += getTransactionSize(tx);
+
+        const signatureAfter = log.signatureAfter[txIdx];
+
+        if (signatureAfter) {
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            signatureAfter,
+            txIdx,
+          );
+          // When we meet a signatureAfter it means that the transaction log exceeds the recommended size
+          // so we move the session content to a dedicated piece, because it must be sent in a standalone piece
+          moveSessionContentToNewPiece(sessionID);
+          currentSessionSize = 0;
+        } else if (isLastItem) {
+          if (!log.lastSignature) {
+            throw new Error(
+              "All the SessionLogs sent must have a lastSignature",
+              {
+                cause: log,
+              },
+            );
           }
-          sessionsTodoAgain.add(sessionID);
-        }
-
-        const oldPieceSize = pieceSize;
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          pieceSize += getTransactionSize(tx);
-        }
 
-        if (exceedsRecommendedSize(pieceSize)) {
-          if (!currentPiece.expectContentUntil && pieces.length === 1) {
-            currentPiece.expectContentUntil =
-              this.knownStateWithStreaming().sessions;
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            log.lastSignature,
+            txIdx,
+          );
+
+          // If the current session size already exceeds the recommended size, we move the session content to a dedicated piece
+          if (exceedsRecommendedSize(currentSessionSize)) {
+            assertLastSignature(sessionID, currentPiece);
+            moveSessionContentToNewPiece(sessionID);
+          } else if (exceedsRecommendedSize(pieceSize, currentSessionSize)) {
+            assertLastSignature(sessionID, currentPiece);
+            startNewPiece();
+          } else {
+            pieceSize += currentSessionSize;
           }
-
-          currentPiece = createContentMessage(this.id, this.header, false);
-          pieces.push(currentPiece);
-          pieceSize = pieceSize - oldPieceSize;
+        } else {
+          // Unsafely add the transaction to the content message, without a signature because we don't have one for this session
+          // Checks and assertions are enforced in this function to avoid that a content message gets out without a signature
+          const signature = undefined as Signature | undefined;
+          addTransactionToContentMessage(
+            currentPiece,
+            tx,
+            sessionID,
+            signature!,
+            txIdx,
+          );
         }
+      }
 
-        let sessionEntry = currentPiece.new[sessionID];
-        if (!sessionEntry) {
-          sessionEntry = {
-            after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
-            newTransactions: [],
-            lastSignature: "WILL_BE_REPLACED" as Signature,
-          };
-          currentPiece.new[sessionID] = sessionEntry;
-        }
+      assertLastSignature(sessionID, currentPiece);
+    }
 
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          sessionEntry.newTransactions.push(tx);
-        }
+    const firstPiece = pieces[0];
 
-        sessionEntry.lastSignature =
-          nextKnownSignatureIdx === undefined
-            ? log.lastSignature!
-            : log.signatureAfter[nextKnownSignatureIdx]!;
+    if (!firstPiece) {
+      throw new Error("First piece not found", {
+        cause: pieces,
+      });
+    }
 
-        sentState[sessionID] =
-          (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
-      }
+    const includeHeader = !knownState?.header;
+
+    if (includeHeader) {
+      firstPiece.header = this.header;
     }
 
     const piecesWithContent = pieces.filter(
-      (piece) => Object.keys(piece.new).length > 0 || piece.header,
+      (piece) => piece.header || Object.keys(piece.new).length > 0,
     );
 
-    if (piecesWithContent.length === 0) {
-      return undefined;
+    if (piecesWithContent.length > 1 || this.isStreaming()) {
+      // Flag that more content is coming
+      if (knownState) {
+        firstPiece.expectContentUntil = getKnownStateToSend(
+          this.knownStateWithStreaming().sessions,
+          knownState.sessions,
+        );
+      } else {
+        firstPiece.expectContentUntil = {
+          ...this.knownStateWithStreaming().sessions,
+        };
+      }
     }
 
-    if (isKnownStateEmpty) {
-      this._cachedNewContentSinceEmpty = piecesWithContent;
+    if (piecesWithContent.length === 0) {
+      return undefined;
     }
 
     return piecesWithContent;
@@ -307,15 +342,10 @@ export class VerifiedState {
   }
 }
 
-function getNextKnownSignatureIdx(
-  log: SessionLog,
-  knownStateForSessionID?: number,
-  sentStateForSessionID?: number,
-) {
-  return Object.keys(log.signatureAfter)
-    .map(Number)
-    .sort((a, b) => a - b)
-    .find(
-      (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
-    );
+function assertLastSignature(sessionID: SessionID, content: NewContentMessage) {
+  if (content.new[sessionID] && !content.new[sessionID].lastSignature) {
+    throw new Error("The SessionContent sent must have a lastSignature", {
+      cause: content.new[sessionID],
+    });
+  }
 }

package/src/coValues/group.ts

@@ -123,6 +123,13 @@ function healMissingKeyForEveryone(group: RawGroup) {
 }
 
 function needsKeyRotation(group: RawGroup) {
+  const myRole = group.myRole();
+
+  // Checking only direct membership because inside the migrations we can't navigate the parent groups
+  if (myRole !== "admin" && myRole !== "manager") {
+    return false;
+  }
+
   const currentReadKeyId = group.get("readKey");
 
   if (!currentReadKeyId) {
@@ -190,11 +197,27 @@ export class RawGroup<
   ) {
     super(core, options);
     this.crypto = core.node.crypto;
+    this.migrate();
+  }
 
-    // Checks if this is not an account
-    if (core.isGroup() && !core.verified.isStreaming()) {
-      // rotateReadKeyIfNeeded(this);
+  migrate() {
+    if (!this.core.isGroup()) {
+      return;
+    }
+
+    const runMigrations = () => {
+      rotateReadKeyIfNeeded(this);
       healMissingKeyForEveryone(this);
+    };
+
+    // We need the group and their parents to be completely downloaded to correctly handle the migrations
+    if (!this.core.isCompletelyDownloaded()) {
+      this.core.waitFor({
+        predicate: (core) => core.isCompletelyDownloaded(),
+        onSuccess: runMigrations,
+      });
+    } else {
+      runMigrations();
     }
   }
 
@@ -318,7 +341,7 @@ export class RawGroup<
    * @category 1. Role reading
    */
   myRole(): Role | undefined {
-    return this.roleOfInternal(this.core.node.getCurrentAgent().id);
+    return this.roleOfInternal(this.core.node.getCurrentAccountOrAgentID());
   }
 
   /**