cojson 0.18.18 → 0.18.20

package/package.json

@@ -29,7 +29,7 @@
   },
   "type": "module",
   "license": "MIT",
-  "version": "0.18.18",
+  "version": "0.18.20",
   "devDependencies": {
     "@opentelemetry/sdk-metrics": "^2.0.0",
     "libsql": "^0.5.13",
@@ -43,7 +43,7 @@
     "@scure/base": "1.2.1",
     "neverthrow": "^7.0.1",
     "unicode-segmenter": "^0.12.0",
-    "cojson-core-wasm": "0.18.18"
+    "cojson-core-wasm": "0.18.20"
   },
   "gitHead": "33c27053293b4801b968c61d5c4c989f93a67d13",
   "scripts": {

package/src/GarbageCollector.ts

@@ -5,7 +5,10 @@ import { RawCoID } from "./ids.js";
 export class GarbageCollector {
   private readonly interval: ReturnType<typeof setInterval>;
 
-  constructor(private readonly coValues: Map<RawCoID, CoValueCore>) {
+  constructor(
+    private readonly coValues: Map<RawCoID, CoValueCore>,
+    private readonly garbageCollectGroups: boolean,
+  ) {
     this.interval = setInterval(() => {
       this.collect();
     }, GARBAGE_COLLECTOR_CONFIG.INTERVAL);
@@ -33,7 +36,7 @@ export class GarbageCollector {
       const timeSinceLastAccessed = currentTime - verified.lastAccessed;
 
       if (timeSinceLastAccessed > GARBAGE_COLLECTOR_CONFIG.MAX_AGE) {
-        coValue.unmount();
+        coValue.unmount(this.garbageCollectGroups);
       }
     }
   }

package/src/coValueCore/coValueCore.ts

@@ -20,10 +20,19 @@ import { JsonObject, JsonValue } from "../jsonValue.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
 import { determineValidTransactions } from "../permissions.js";
-import { CoValueKnownState, PeerID, emptyKnownState } from "../sync.js";
+import {
+  CoValueKnownState,
+  NewContentMessage,
+  PeerID,
+  emptyKnownState,
+} from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
-import { getDependedOnCoValuesFromRawData } from "./utils.js";
+import {
+  getDependenciesFromContentMessage,
+  getDependenciesFromGroupRawTransactions,
+  getDependenciesFromHeader,
+} from "./utils.js";
 import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 import { SessionMap } from "./SessionMap.js";
 import {
@@ -125,7 +134,6 @@ export class CoValueCore {
   private _cachedContent?: RawCoValue;
   readonly listeners: Set<(core: CoValueCore, unsub: () => void) => void> =
     new Set();
-  private _cachedDependentOn?: Set<RawCoID>;
   private counter: UpDownCounter;
 
   private constructor(
@@ -182,8 +190,12 @@ export class CoValueCore {
     return "unavailable";
   }
 
+  hasMissingDependencies() {
+    return this.missingDependencies.size > 0;
+  }
+
   isAvailable(): this is AvailableCoValueCore {
-    return this.hasVerifiedContent() && this.missingDependencies.size === 0;
+    return this.hasVerifiedContent();
   }
 
   hasVerifiedContent(): this is AvailableCoValueCore {
@@ -251,9 +263,11 @@ export class CoValueCore {
     }
   }
 
-  unmount() {
-    // For simplicity, we don't unmount groups and accounts
-    if (this.verified?.header.ruleset.type === "group") {
+  unmount(garbageCollectGroups = false) {
+    if (
+      !garbageCollectGroups &&
+      this.verified?.header.ruleset.type === "group"
+    ) {
       return false;
     }
 
@@ -280,10 +294,15 @@ export class CoValueCore {
     this.scheduleNotifyUpdate();
   }
 
+  markFoundInPeer(peerId: PeerID, previousState: string) {
+    this.peers.set(peerId, { type: "available" });
+    this.updateCounter(previousState);
+    this.scheduleNotifyUpdate();
+  }
+
   missingDependencies = new Set<RawCoID>();
 
-  // Checks if the current CoValueCore is already a missing dependency of the given CoValueCore
-  checkCircularDependencies(dependency: CoValueCore) {
+  isCircularMissingDependency(dependency: CoValueCore) {
     const visited = new Set<RawCoID>();
     const stack = [dependency];
 
@@ -291,14 +310,14 @@ export class CoValueCore {
       const current = stack.pop();
 
       if (!current) {
-        return true;
+        return false;
       }
 
       visited.add(current.id);
 
       for (const dependency of current.missingDependencies) {
         if (dependency === this.id) {
-          return false;
+          return true;
         }
 
         if (!visited.has(dependency)) {
@@ -307,43 +326,71 @@ export class CoValueCore {
       }
     }
 
-    return true;
+    return false;
   }
 
-  markMissingDependency(dependency: RawCoID) {
-    const value = this.node.getCoValue(dependency);
+  markDependencyAvailable(dependency: RawCoID) {
+    this.missingDependencies.delete(dependency);
 
-    if (value.isAvailable()) {
-      this.missingDependencies.delete(dependency);
-    } else if (this.checkCircularDependencies(value)) {
-      const unsubscribe = value.subscribe(() => {
-        if (value.isAvailable()) {
-          this.missingDependencies.delete(dependency);
-          unsubscribe();
-        }
+    if (this.missingDependencies.size === 0) {
+      this.scheduleNotifyUpdate();
+    }
+  }
 
-        if (this.isAvailable()) {
-          this.scheduleNotifyUpdate();
+  newContentQueue: {
+    msg: NewContentMessage;
+    from: PeerState | "storage" | "import";
+  }[] = [];
+  /**
+   * Add a new content to the queue and handle it when the dependencies are available
+   */
+  addNewContentToQueue(
+    msg: NewContentMessage,
+    from: PeerState | "storage" | "import",
+  ) {
+    const alreadyEnqueued = this.newContentQueue.length > 0;
+
+    this.newContentQueue.push({ msg, from });
+
+    if (alreadyEnqueued) {
+      return;
+    }
+
+    this.subscribe((core, unsubscribe) => {
+      if (!core.hasMissingDependencies()) {
+        unsubscribe();
+
+        const enqueuedNewContent = this.newContentQueue;
+        this.newContentQueue = [];
+
+        for (const { msg, from } of enqueuedNewContent) {
+          this.node.syncManager.handleNewContent(msg, from);
         }
-      });
+      }
+    });
+  }
 
-      this.missingDependencies.add(dependency);
+  addDependencyFromHeader(header: CoValueHeader) {
+    for (const dep of getDependenciesFromHeader(header)) {
+      this.addDependency(dep);
     }
   }
 
   provideHeader(
     header: CoValueHeader,
-    fromPeerId: PeerID,
     streamingKnownState?: CoValueKnownState["sessions"],
+    skipVerify?: boolean,
   ) {
-    const previousState = this.loadingState;
-
-    const expectedId = idforHeader(header, this.node.crypto);
+    if (!skipVerify) {
+      const expectedId = idforHeader(header, this.node.crypto);
 
-    if (this.id !== expectedId) {
-      return false;
+      if (this.id !== expectedId) {
+        return false;
+      }
     }
 
+    this.addDependencyFromHeader(header);
+
     if (this._verified?.sessions.size) {
       throw new Error(
         "CoValueCore: provideHeader called on coValue with verified sessions present!",
@@ -357,26 +404,9 @@ export class CoValueCore {
       streamingKnownState,
     );
 
-    this.peers.set(fromPeerId, { type: "available" });
-
-    this.updateCounter(previousState);
-    this.scheduleNotifyUpdate();
-
     return true;
   }
 
-  internalMarkMagicallyAvailable(
-    verified: VerifiedState,
-    { forceOverwrite = false }: { forceOverwrite?: boolean } = {},
-  ) {
-    const previousState = this.loadingState;
-    this.internalShamefullyCloneVerifiedStateFrom(verified, {
-      forceOverwrite,
-    });
-    this.updateCounter(previousState);
-    this.scheduleNotifyUpdate();
-  }
-
   markErrored(peerId: PeerID, error: TryAddTransactionsError) {
     const previousState = this.loadingState;
     this.peers.set(peerId, { type: "errored", error });
@@ -391,25 +421,6 @@ export class CoValueCore {
     this.scheduleNotifyUpdate();
   }
 
-  internalShamefullyCloneVerifiedStateFrom(
-    state: VerifiedState,
-    { forceOverwrite = false }: { forceOverwrite?: boolean } = {},
-  ) {
-    if (!forceOverwrite && this._verified?.sessions.size) {
-      throw new Error(
-        "CoValueCore: internalShamefullyCloneVerifiedStateFrom called on coValue with verified sessions present!",
-      );
-    }
-    this._verified = state.clone();
-    this.internalShamefullyResetCachedContent();
-  }
-
-  internalShamefullyResetCachedContent() {
-    this._cachedContent = undefined;
-    this._cachedDependentOn = undefined;
-    this.resetParsedTransactions();
-  }
-
   groupInvalidationSubscription?: () => void;
 
   subscribeToGroupInvalidation() {
@@ -430,7 +441,7 @@ export class CoValueCore {
       if (entry.isAvailable()) {
         this.groupInvalidationSubscription = entry.subscribe((_groupUpdate) => {
           // When the group is updated, we need to reset the cached content because the transactions validity might have changed
-          this.internalShamefullyResetCachedContent();
+          this.resetParsedTransactions();
           this.scheduleNotifyUpdate();
         }, false);
       } else {
@@ -445,7 +456,7 @@ export class CoValueCore {
   contentInClonedNodeWithDifferentAccount(account: ControlledAccountOrAgent) {
     return this.node
       .loadCoValueAsDifferentAgent(this.id, account.agentSecret, account.id)
-      .getCurrentContent();
+      .then((core) => core.getCurrentContent());
   }
 
   knownStateWithStreaming(): CoValueKnownState {
@@ -490,6 +501,14 @@ export class CoValueCore {
     };
   }
 
+  addDependenciesFromContentMessage(newContent: NewContentMessage) {
+    const dependencies = getDependenciesFromContentMessage(this, newContent);
+
+    for (const dependency of dependencies) {
+      this.addDependency(dependency);
+    }
+  }
+
   tryAddTransactions(
     sessionID: SessionID,
     newTransactions: Transaction[],
@@ -528,7 +547,7 @@ export class CoValueCore {
       );
 
       if (result.isOk()) {
-        this.updateCurrentContent();
+        this.processNewTransactions();
         this.scheduleNotifyUpdate();
       }
 
@@ -536,18 +555,18 @@ export class CoValueCore {
     });
   }
 
-  private updateCurrentContent() {
-    if (
-      this._cachedContent &&
-      "processNewTransactions" in this._cachedContent &&
-      typeof this._cachedContent.processNewTransactions === "function"
-    ) {
-      this._cachedContent.processNewTransactions();
-    } else {
-      this._cachedContent = undefined;
+  private processNewTransactions() {
+    if (this._cachedContent) {
+      // Does the cached content support incremental processing?
+      if (
+        "processNewTransactions" in this._cachedContent &&
+        typeof this._cachedContent.processNewTransactions === "function"
+      ) {
+        this._cachedContent.processNewTransactions();
+      } else {
+        this._cachedContent = undefined;
+      }
     }
-
-    this._cachedDependentOn = undefined;
   }
 
   #isNotificationScheduled = false;
@@ -574,8 +593,18 @@ export class CoValueCore {
     }
   }
 
+  #isNotifyUpdatePaused = false;
+  pauseNotifyUpdate() {
+    this.#isNotifyUpdatePaused = true;
+  }
+
+  resumeNotifyUpdate() {
+    this.#isNotifyUpdatePaused = false;
+    this.notifyUpdate();
+  }
+
   private notifyUpdate() {
-    if (this.listeners.size === 0) {
+    if (this.listeners.size === 0 || this.#isNotifyUpdatePaused) {
       return;
     }
 
@@ -664,15 +693,17 @@ export class CoValueCore {
 
     const { transaction, signature } = result;
 
-    this.node.syncManager.recordTransactionsSize([transaction], "local");
+    // Assign pre-parsed meta and changes to skip the parse/decrypt operation when loading
+    // this transaction in the current content
+    this.parsingCache.set(transaction, { changes, meta });
 
-    // We pre-populate the parsed transactions and meta for the new transaction, to skip the parsing step later
-    this.loadVerifiedTransactionsFromLogs({ transaction, changes, meta });
+    this.node.syncManager.recordTransactionsSize([transaction], "local");
 
     const session = this.verified.sessions.get(sessionID);
     const txIdx = session ? session.transactions.length - 1 : 0;
 
-    this.updateCurrentContent();
+    this.processNewTransactions();
+    this.addDependenciesFromNewTransaction(transaction);
 
     // force immediate notification because local updates may come from the UI
     // where we need synchronous updates
@@ -688,6 +719,16 @@ export class CoValueCore {
     return true;
   }
 
+  addDependenciesFromNewTransaction(transaction: Transaction) {
+    if (this.verified?.header.ruleset.type === "group") {
+      for (const dependency of getDependenciesFromGroupRawTransactions([
+        transaction,
+      ])) {
+        this.addDependency(dependency);
+      }
+    }
+  }
+
   getCurrentContent(options?: { ignorePrivateTransactions: true }): RawCoValue {
     if (!this.verified) {
       throw new Error(
@@ -721,6 +762,8 @@ export class CoValueCore {
 
   // Reset the parsed transactions and branches, to validate them again from scratch when the group is updated
   resetParsedTransactions() {
+    this._cachedContent = undefined;
+
     this.branchStart = undefined;
     this.mergeCommits = [];
 
@@ -738,23 +781,21 @@ export class CoValueCore {
     VerifiedTransaction
   > = {};
 
+  private parsingCache = new Map<
+    Transaction,
+    { changes: JsonValue[]; meta: JsonObject | undefined }
+  >();
+
   /**
    * Loads the new transaction from the SessionMap into verifiedTransactions as a VerifiedTransaction.
    *
    * If the transaction is already loaded from the SessionMap in the past, it will not be loaded again.
    *
    * Used to have a fast way to iterate over the CoValue transactions, and track their validation/decoding state.
-   *
-   * @param preload - Optional preload object containing the transaction, changes, and meta.
-   * If provided, the transaction will be preloaded with the given changes and meta.
-   *
-   * @internal
+
+  * @internal
    * */
-  loadVerifiedTransactionsFromLogs(preload?: {
-    transaction: Transaction;
-    changes: JsonValue[];
-    meta: JsonObject | undefined;
-  }) {
+  loadVerifiedTransactionsFromLogs() {
     if (!this.verified) {
       return;
     }
@@ -780,14 +821,19 @@ export class CoValueCore {
               txIndex,
             };
 
+        const cache = this.parsingCache.get(tx);
+        if (cache) {
+          this.parsingCache.delete(tx);
+        }
+
         const verifiedTransaction = {
           author: accountOrAgentIDfromSessionID(sessionID),
           txID,
           madeAt: tx.madeAt,
           isValidated: false,
           isValid: false,
-          changes: tx === preload?.transaction ? preload.changes : undefined,
-          meta: tx === preload?.transaction ? preload.meta : undefined,
+          changes: cache?.changes,
+          meta: cache?.meta,
           hasInvalidChanges: false,
           hasInvalidMeta: false,
           hasMetaBeenParsed: false,
@@ -978,6 +1024,32 @@ export class CoValueCore {
     return matchingTransactions;
   }
 
+  dependencies: Set<RawCoID> = new Set();
+  private addDependency(dependency: RawCoID) {
+    if (this.dependencies.has(dependency)) {
+      return true;
+    }
+
+    this.dependencies.add(dependency);
+
+    const dependencyCoValue = this.node.getCoValue(dependency);
+
+    if (
+      !dependencyCoValue.isAvailable() &&
+      !this.isCircularMissingDependency(dependencyCoValue)
+    ) {
+      this.missingDependencies.add(dependency);
+
+      dependencyCoValue.subscribe((dependencyCoValue, unsubscribe) => {
+        if (dependencyCoValue.isAvailable()) {
+          unsubscribe();
+          this.markDependencyAvailable(dependency);
+        }
+      });
+      return false;
+    }
+  }
+
   createBranch(name: string, ownerId?: RawCoID) {
     return createBranch(this, name, ownerId);
   }
@@ -1144,25 +1216,7 @@ export class CoValueCore {
   }
 
   getDependedOnCoValues(): Set<RawCoID> {
-    if (this._cachedDependentOn) {
-      return this._cachedDependentOn;
-    } else {
-      if (!this.verified) {
-        return new Set();
-      }
-
-      const dependentOn = getDependedOnCoValuesFromRawData(
-        this.id,
-        this.verified.header,
-        this.verified.sessions.keys(),
-        Array.from(
-          this.verified.sessions.values(),
-          (session) => session.transactions,
-        ),
-      );
-      this._cachedDependentOn = dependentOn;
-      return dependentOn;
-    }
+    return this.dependencies;
   }
 
   waitForSync(options?: { timeout?: number }) {

package/src/coValueCore/utils.ts

@@ -1,61 +1,115 @@
+import { CoValueCore } from "../exports.js";
 import { getGroupDependentKey } from "../ids.js";
 import { RawCoID, SessionID } from "../ids.js";
 import { Stringified, parseJSON } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
+import { NewContentMessage } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isAccountID } from "../typeUtils/isAccountID.js";
 import { CoValueHeader, Transaction } from "./verifiedState.js";
 
-export function getDependedOnCoValuesFromRawData(
-  id: RawCoID,
+export function getDependenciesFromHeader(
   header: CoValueHeader,
-  sessions: Iterable<SessionID>,
-  transactions: Iterable<Iterable<Transaction>>,
+  deps = new Set<RawCoID>(),
 ): Set<RawCoID> {
-  const deps = new Set<RawCoID>();
+  if (header.ruleset.type === "ownedByGroup") {
+    deps.add(header.ruleset.group);
+  }
 
+  if (header.meta?.source) {
+    deps.add(header.meta.source as RawCoID);
+  }
+
+  return deps;
+}
+
+export function getDependenciesFromSessions(
+  sessions: Iterable<SessionID>,
+  deps: Set<RawCoID>,
+): void {
   for (const session of sessions) {
     const accountId = accountOrAgentIDfromSessionID(session);
 
-    if (isAccountID(accountId) && accountId !== id) {
+    if (isAccountID(accountId)) {
       deps.add(accountId);
     }
   }
+}
 
-  if (header.ruleset.type === "group") {
-    for (const txs of transactions) {
-      for (const tx of txs) {
-        if (tx.privacy !== "trusting") continue;
-
-        const changes = safeParseChanges(tx.changes);
-        for (const change of changes) {
-          if (
-            change &&
-            typeof change === "object" &&
-            "op" in change &&
-            change.op === "set" &&
-            "key" in change &&
-            change.key
-          ) {
-            const key = getGroupDependentKey(change.key);
-
-            if (key && key !== id) {
-              deps.add(key);
-            }
-          }
+export function getDependenciesFromGroupRawTransactions(
+  transactions: Iterable<Transaction>,
+  deps = new Set<RawCoID>(),
+): Set<RawCoID> {
+  for (const tx of transactions) {
+    if (tx.privacy !== "trusting") continue;
+
+    const changes = safeParseChanges(tx.changes);
+    for (const change of changes) {
+      if (
+        change &&
+        typeof change === "object" &&
+        "op" in change &&
+        change.op === "set" &&
+        "key" in change &&
+        change.key
+      ) {
+        const key = getGroupDependentKey(change.key);
+
+        if (key) {
+          deps.add(key);
         }
       }
     }
   }
 
-  if (header.ruleset.type === "ownedByGroup") {
-    deps.add(header.ruleset.group);
+  return deps;
+}
+
+export function getDependedOnCoValuesFromRawData(
+  id: RawCoID,
+  header: CoValueHeader,
+  sessions: Iterable<SessionID>,
+  transactions: Iterable<Iterable<Transaction>>,
+): Set<RawCoID> {
+  const deps = new Set<RawCoID>();
+
+  getDependenciesFromHeader(header, deps);
+  getDependenciesFromSessions(sessions, deps);
+
+  if (header.ruleset.type === "group") {
+    for (const txs of transactions) {
+      getDependenciesFromGroupRawTransactions(txs, deps);
+    }
   }
 
-  if (header.meta?.source) {
-    deps.add(header.meta.source as RawCoID);
+  deps.delete(id);
+
+  return deps;
+}
+
+export function getDependenciesFromContentMessage(
+  coValue: CoValueCore,
+  contentMessage: NewContentMessage,
+) {
+  const deps = new Set<RawCoID>();
+
+  if (contentMessage.header) {
+    getDependenciesFromHeader(contentMessage.header, deps);
   }
 
+  const sessions = Object.keys(contentMessage.new) as SessionID[];
+  getDependenciesFromSessions(sessions, deps);
+
+  const header = coValue.verified?.header ?? contentMessage.header;
+
+  if (header?.ruleset.type === "group") {
+    for (const { newTransactions } of Object.values(contentMessage.new)) {
+      getDependenciesFromGroupRawTransactions(newTransactions, deps);
+    }
+  }
+
+  deps.delete(coValue.id);
+
   return deps;
 }