cojson 0.18.29 → 0.18.31

package/src/coValueCore/coValueCore.ts

@@ -44,6 +44,7 @@ import {
 import { type RawAccountID } from "../coValues/account.js";
 import { decryptTransactionChangesAndMeta } from "./decryptTransactionChangesAndMeta.js";
 import {
+  cloneKnownState,
   combineKnownStateSessions,
   CoValueKnownState,
   emptyKnownState,
@@ -60,6 +61,7 @@ export function idforHeader(
 }
 
 export class VerifiedTransaction {
+  dispatchTransaction: (transaction: VerifiedTransaction) => void;
   // The account or agent that made the transaction
   author: RawAccountID | AgentID;
   // An object containing the session ID and the transaction index
@@ -75,14 +77,9 @@ export class VerifiedTransaction {
   changes: JsonValue[] | undefined;
   // The decoded meta information of the transaction
   meta: JsonObject | undefined;
+  isValidated: boolean = false;
   // Whether the transaction is valid, as per membership rules
   isValid: boolean = false;
-  // Whether the transaction has been validated, used to track if determinedValidTransactions needs to check this
-  isValidated: boolean = false;
-  // True if the transaction has been decrypted
-  isDecrypted: boolean = false;
-  // True if the meta information has been parsed and loaded in the CoValueCore
-  hasMetaBeenParsed: boolean = false;
   // The previous verified transaction for the same session
   previous: VerifiedTransaction | undefined;
 
@@ -95,7 +92,9 @@ export class VerifiedTransaction {
       | { changes: JsonValue[]; meta: JsonObject | undefined }
       | undefined,
     previous: VerifiedTransaction | undefined,
+    dispatchTransaction: (transaction: VerifiedTransaction) => void,
   ) {
+    this.dispatchTransaction = dispatchTransaction;
     this.author = accountOrAgentIDfromSessionID(sessionID);
 
     const txID = branchId
@@ -114,7 +113,6 @@ export class VerifiedTransaction {
     this.tx = tx;
     this.currentMadeAt = tx.madeAt;
     this.sourceTxMadeAt = undefined;
-    this.isValidated = false;
 
     this.previous = previous;
 
@@ -151,6 +149,20 @@ export class VerifiedTransaction {
   } {
     return Boolean(this.isValid && this.changes);
   }
+
+  markValid() {
+    this.isValid = true;
+
+    if (!this.isValidated) {
+      this.isValidated = true;
+      this.dispatchTransaction(this);
+    }
+  }
+
+  markInvalid() {
+    this.isValidated = true;
+    this.isValid = false;
+  }
 }
 
 export type DecryptedTransaction = Omit<VerifiedTransaction, "changes"> & {
@@ -242,6 +254,31 @@ export class CoValueCore {
     return this.hasVerifiedContent();
   }
 
+  /**
+   * True if the coValue is completely downloaded:
+   * - the current coValue is available and not streaming
+   * - the group is available and not streaming
+   * - TODO: all the parent groups are available and not streaming
+   */
+  isCompletelyDownloaded(): this is AvailableCoValueCore {
+    if (!this.hasVerifiedContent()) {
+      return false;
+    }
+
+    if (this.verified.isStreaming()) {
+      return false;
+    }
+
+    const group = this.safeGetGroup();
+
+    // TODO: Group coValues should be completely downloaded when all their parent groups are completely downloaded
+    if (!group) {
+      return true;
+    }
+
+    return group.core.isCompletelyDownloaded();
+  }
+
   hasVerifiedContent(): this is AvailableCoValueCore {
     return !!this.verified;
   }
@@ -432,6 +469,8 @@ export class CoValueCore {
       new SessionMap(this.id, this.node.crypto, streamingKnownState),
     );
 
+    this.resetKnownStateCache();
+
     return true;
   }
 
@@ -487,20 +526,46 @@ export class CoValueCore {
       .then((core) => core.getCurrentContent());
   }
 
+  private _cachedKnownStateWithStreaming?: CoValueKnownState;
+  /**
+   * Returns the known state considering the known state of the streaming source
+   *
+   * Used to correctly manage the content & subscriptions during the content streaming process
+   */
   knownStateWithStreaming(): CoValueKnownState {
-    if (this.verified) {
-      return this.verified.knownStateWithStreaming();
-    } else {
-      return emptyKnownState(this.id);
+    if (this._cachedKnownStateWithStreaming) {
+      return this._cachedKnownStateWithStreaming;
     }
+
+    const knownState = this.verified
+      ? cloneKnownState(this.verified.knownStateWithStreaming())
+      : emptyKnownState(this.id);
+
+    this._cachedKnownStateWithStreaming = knownState;
+
+    return knownState;
   }
 
+  private _cachedKnownState?: CoValueKnownState;
+  /**
+   * Returns the known state of the CoValue
+   *
+   * The return value identity is going to be stable as long as the CoValue is not modified.
+   *
+   * On change the knownState is invalidated and a new object is returned.
+   */
   knownState(): CoValueKnownState {
-    if (this.verified) {
-      return this.verified.knownState();
-    } else {
-      return emptyKnownState(this.id);
+    if (this._cachedKnownState) {
+      return this._cachedKnownState;
     }
+
+    const knownState = this.verified
+      ? cloneKnownState(this.verified.knownState())
+      : emptyKnownState(this.id);
+
+    this._cachedKnownState = knownState;
+
+    return knownState;
   }
 
   get meta(): JsonValue {
@@ -575,6 +640,7 @@ export class CoValueCore {
       );
 
       if (result.isOk()) {
+        this.resetKnownStateCache();
         this.processNewTransactions();
         this.scheduleNotifyUpdate();
       }
@@ -583,6 +649,11 @@ export class CoValueCore {
     });
   }
 
+  private resetKnownStateCache() {
+    this._cachedKnownState = undefined;
+    this._cachedKnownStateWithStreaming = undefined;
+  }
+
   private processNewTransactions() {
     if (this._cachedContent) {
       // Does the cached content support incremental processing?
@@ -730,6 +801,7 @@ export class CoValueCore {
     const session = this.verified.sessions.get(sessionID);
     const txIdx = session ? session.transactions.length - 1 : 0;
 
+    this.resetKnownStateCache();
     this.processNewTransactions();
     this.addDependenciesFromNewTransaction(transaction);
 
@@ -768,10 +840,10 @@ export class CoValueCore {
       return this._cachedContent;
     }
 
-    this.subscribeToGroupInvalidation();
-
     const newContent = coreToCoValue(this as AvailableCoValueCore, options);
 
+    this.subscribeToGroupInvalidation();
+
     if (!options?.ignorePrivateTransactions) {
       this._cachedContent = newContent;
     }
@@ -797,11 +869,20 @@ export class CoValueCore {
 
     for (const transaction of this.verifiedTransactions) {
       transaction.isValidated = false;
-      transaction.hasMetaBeenParsed = false;
     }
+
+    this.toValidateTransactions = this.verifiedTransactions.slice();
+    this.toProcessTransactions = [];
+    this.toDecryptTransactions = [];
+    this.toParseMetaTransactions = [];
   }
 
   verifiedTransactions: VerifiedTransaction[] = [];
+  toValidateTransactions: VerifiedTransaction[] = [];
+  toDecryptTransactions: VerifiedTransaction[] = [];
+  toParseMetaTransactions: VerifiedTransaction[] = [];
+  toProcessTransactions: VerifiedTransaction[] = [];
+
   private verifiedTransactionsKnownSessions: CoValueKnownState["sessions"] = {};
 
   private lastVerifiedTransactionBySessionID: Record<
@@ -855,6 +936,7 @@ export class CoValueCore {
           isBranched ? this.id : undefined,
           cache,
           this.lastVerifiedTransactionBySessionID[sessionID],
+          this.dispatchTransaction,
         );
 
         if (verifiedTransaction.madeAt > this.latestTxMadeAt) {
@@ -866,6 +948,7 @@ export class CoValueCore {
         }
 
         this.verifiedTransactions.push(verifiedTransaction);
+        this.dispatchTransaction(verifiedTransaction);
         this.lastVerifiedTransactionBySessionID[sessionID] =
           verifiedTransaction;
       }
@@ -875,27 +958,39 @@ export class CoValueCore {
     }
   }
 
+  dispatchTransaction = (transaction: VerifiedTransaction) => {
+    if (!transaction.isValidated) {
+      this.toValidateTransactions.push(transaction);
+      return;
+    }
+
+    if (transaction.changes) {
+      this.toProcessTransactions.push(transaction);
+    } else {
+      this.toDecryptTransactions.push(transaction);
+    }
+
+    if (transaction.meta) {
+      this.toParseMetaTransactions.push(transaction);
+    }
+  };
+
   /**
    * Iterates over the verifiedTransactions and marks them as valid or invalid, based on the group membership of the authors of the transactions  .
    */
   private determineValidTransactions() {
     determineValidTransactions(this);
+    this.toValidateTransactions = [];
   }
 
   /**
    * Parses the meta information of a transaction, and set the branchStart and mergeCommits.
    */
   private parseMetaInformation(transaction: VerifiedTransaction) {
-    if (
-      !transaction.meta ||
-      !transaction.isValid ||
-      transaction.hasMetaBeenParsed
-    ) {
+    if (!transaction.meta) {
       return;
     }
 
-    transaction.hasMetaBeenParsed = true;
-
     // Branch related meta information
     if (this.isBranched()) {
       // Check if the transaction is a branch start
@@ -945,7 +1040,7 @@ export class CoValueCore {
         transaction.sourceTxMadeAt &&
         transaction.sourceTxMadeAt > transaction.currentMadeAt
       ) {
-        transaction.isValid = false;
+        transaction.markInvalid();
       }
 
       if (sessionID) {
@@ -973,15 +1068,21 @@ export class CoValueCore {
     if (!this.isAvailable()) {
       return;
     }
-
     this.loadVerifiedTransactionsFromLogs();
     this.determineValidTransactions();
 
-    for (const transaction of this.verifiedTransactions) {
-      if (!ignorePrivateTransactions) {
+    if (!ignorePrivateTransactions) {
+      const toDecryptTransactions = this.toDecryptTransactions;
+      this.toDecryptTransactions = [];
+      for (const transaction of toDecryptTransactions) {
         decryptTransactionChangesAndMeta(this, transaction);
+        this.dispatchTransaction(transaction);
       }
+    }
 
+    const toParseMetaTransactions = this.toParseMetaTransactions;
+    this.toParseMetaTransactions = [];
+    for (const transaction of toParseMetaTransactions) {
       this.parseMetaInformation(transaction);
     }
   }
@@ -994,9 +1095,7 @@ export class CoValueCore {
     // The range, described as knownState sessions, to filter the transactions returned
     from?: CoValueKnownState["sessions"];
     to?: CoValueKnownState["sessions"];
-
-    // The transactions that have already been processed, used for the incremental builds of the content views
-    knownTransactions?: Set<Transaction>;
+    knownTransactions?: Record<RawCoID, number>;
 
     // If true, the branch source transactions will be skipped. Used to gather the transactions for the merge operation.
     skipBranchSource?: boolean;
@@ -1011,39 +1110,48 @@ export class CoValueCore {
 
     const source = getBranchSource(this);
 
-    for (const transaction of this.verifiedTransactions) {
-      if (!transaction.isValidTransactionWithChanges()) {
-        continue;
-      }
+    const from = options?.from;
+    const to = options?.to;
 
-      if (options?.knownTransactions?.has(transaction.tx)) {
+    const knownTransactions = options?.knownTransactions?.[this.id] ?? 0;
+
+    for (
+      let i = knownTransactions;
+      i < this.toProcessTransactions.length;
+      i++
+    ) {
+      const transaction = this.toProcessTransactions[i]!;
+
+      if (!transaction.isValidTransactionWithChanges()) {
         continue;
       }
 
-      options?.knownTransactions?.add(transaction.tx);
-
       // Using the currentTxID to filter the transactions, because the TxID is modified by the merge meta
       const txID = transaction.currentTxID;
 
-      const from = options?.from?.[txID.sessionID] ?? -1;
+      const fromIdx = from?.[txID.sessionID] ?? -1;
 
       // Load the to filter index. Sessions that are not in the to filter will be skipped
-      const to = options?.to ? (options.to[txID.sessionID] ?? -1) : Infinity;
+      const toIdx = to?.[txID.sessionID] ?? Infinity;
 
       // The txIndex starts at 0 and from/to are referring to the count of transactions
-      if (from > txID.txIndex || to < txID.txIndex) {
+      if (fromIdx > txID.txIndex || toIdx < txID.txIndex) {
         continue;
       }
 
       matchingTransactions.push(transaction);
     }
 
+    if (options?.knownTransactions !== undefined) {
+      options.knownTransactions[this.id] = this.toProcessTransactions.length;
+    }
+
     // If this is a branch, we load the valid transactions from the source
     if (source && this.branchStart && !options?.skipBranchSource) {
       const sourceTransactions = source.getValidTransactions({
+        knownTransactions: options?.knownTransactions,
         to: this.branchStart,
         ignorePrivateTransactions: options?.ignorePrivateTransactions ?? false,
-        knownTransactions: options?.knownTransactions,
       });
 
       for (const transaction of sourceTransactions) {
@@ -1064,12 +1172,14 @@ export class CoValueCore {
 
     const dependencyCoValue = this.node.getCoValue(dependency);
 
-    if (
-      !dependencyCoValue.isAvailable() &&
-      !this.isCircularMissingDependency(dependencyCoValue)
-    ) {
-      this.missingDependencies.add(dependency);
+    if (this.isCircularMissingDependency(dependencyCoValue)) {
+      return true;
+    }
+
+    dependencyCoValue.addDependant(this.id);
 
+    if (!dependencyCoValue.isAvailable()) {
+      this.missingDependencies.add(dependency);
       dependencyCoValue.subscribe((dependencyCoValue, unsubscribe) => {
         if (dependencyCoValue.isAvailable()) {
           unsubscribe();
@@ -1080,6 +1190,27 @@ export class CoValueCore {
     }
   }
 
+  dependant: Set<RawCoID> = new Set();
+  private addDependant(dependant: RawCoID) {
+    this.dependant.add(dependant);
+  }
+
+  isGroup() {
+    if (!this.verified) {
+      return false;
+    }
+
+    if (this.verified.header.ruleset.type !== "group") {
+      return false;
+    }
+
+    if (this.verified.header.meta?.type === "account") {
+      return false;
+    }
+
+    return true;
+  }
+
   createBranch(name: string, ownerId?: RawCoID) {
     return createBranch(this, name, ownerId);
   }
@@ -1137,7 +1268,7 @@ export class CoValueCore {
     ignorePrivateTransactions: boolean;
 
     // The transactions that have already been processed, used for the incremental builds of the content views
-    knownTransactions?: Set<Transaction>;
+    knownTransactions?: Record<RawCoID, number>;
   }): DecryptedTransaction[] {
     const allTransactions = this.getValidTransactions(options);
 

package/src/coValueCore/decryptTransactionChangesAndMeta.ts

@@ -6,7 +6,6 @@ export function decryptTransactionChangesAndMeta(
 ) {
   if (
     !transaction.isValid ||
-    transaction.isDecrypted ||
     transaction.tx.privacy === "trusting" // Trusting transactions are already decrypted
   ) {
     return;
@@ -48,9 +47,4 @@ export function decryptTransactionChangesAndMeta(
       transaction.meta = meta;
     }
   }
-
-  // We mark the transaction as decrypted even if the changes or meta have failed to be decrypted
-  // This is because, if we successfully extracted the readKey and the decrypt failed once it will always fail
-  // so better to log the error (we already do that) and mark the transaction as decrypted
-  transaction.isDecrypted = true;
 }

package/src/coValueCore/verifiedState.ts

@@ -55,8 +55,6 @@ export class VerifiedState {
   readonly crypto: CryptoProvider;
   readonly header: CoValueHeader;
   readonly sessions: SessionMap;
-  private _cachedKnownState?: CoValueKnownState;
-  private _cachedKnownStateWithStreaming?: CoValueKnownState;
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   public lastAccessed: number | undefined;
   public branchSourceId?: RawCoID;
@@ -102,8 +100,6 @@ export class VerifiedState {
 
     if (result.isOk()) {
       this._cachedNewContentSinceEmpty = undefined;
-      this._cachedKnownState = undefined;
-      this._cachedKnownStateWithStreaming = undefined;
     }
 
     return result;
@@ -125,8 +121,6 @@ export class VerifiedState {
     );
 
     this._cachedNewContentSinceEmpty = undefined;
-    this._cachedKnownState = undefined;
-    this._cachedKnownStateWithStreaming = undefined;
 
     return result;
   }
@@ -151,8 +145,6 @@ export class VerifiedState {
     );
 
     this._cachedNewContentSinceEmpty = undefined;
-    this._cachedKnownState = undefined;
-    this._cachedKnownStateWithStreaming = undefined;
 
     return result;
   }
@@ -286,39 +278,12 @@ export class VerifiedState {
     return piecesWithContent;
   }
 
-  /**
-   * Returns the known state of the CoValue
-   *
-   * The return value identity is going to be stable as long as the CoValue is not modified.
-   *
-   * On change the knownState is invalidated and a new object is returned.
-   */
   knownState() {
-    if (this._cachedKnownState) {
-      return this._cachedKnownState;
-    }
-    this._cachedKnownState = cloneKnownState(this.sessions.knownState);
-    return this._cachedKnownState;
+    return this.sessions.knownState;
   }
 
-  /**
-   * Returns the known state considering the known state of the streaming source
-   *
-   * Used to correctly manage the content & subscriptions during the content streaming process
-   */
   knownStateWithStreaming() {
-    if (!this.sessions.knownStateWithStreaming) {
-      return this.knownState();
-    }
-
-    if (this._cachedKnownStateWithStreaming) {
-      return this._cachedKnownStateWithStreaming;
-    }
-    this._cachedKnownStateWithStreaming = cloneKnownState(
-      this.sessions.knownStateWithStreaming,
-    );
-
-    return this._cachedKnownStateWithStreaming;
+    return this.sessions.knownStateWithStreaming ?? this.knownState();
   }
 
   isStreaming(): boolean {

package/src/coValues/coList.ts

@@ -1,6 +1,6 @@
 import { CoID, RawCoValue } from "../coValue.js";
 import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
-import { AgentID, SessionID, TransactionID } from "../ids.js";
+import { AgentID, SessionID, TransactionID, RawCoID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
@@ -85,11 +85,8 @@ export class RawCoList<
     opID: OpID;
   }[];
   /** @internal */
-  knownTransactions: Set<Transaction>;
-
-  get totalValidTransactions() {
-    return this.knownTransactions.size;
-  }
+  knownTransactions: Record<RawCoID, number>;
+  totalValidTransactions: number = 0;
 
   lastValidTransaction: number | undefined;
 
@@ -102,7 +99,7 @@ export class RawCoList<
     this.deletionsByInsertion = {};
     this.afterStart = [];
     this.beforeEnd = [];
-    this.knownTransactions = new Set<Transaction>();
+    this.knownTransactions = { [core.id]: 0 };
 
     this.processNewTransactions();
   }
@@ -282,6 +279,8 @@ export class RawCoList<
     } else {
       this.lastValidTransaction = lastValidTransaction;
     }
+
+    this.totalValidTransactions += transactions.length;
   }
 
   /** @category 6. Meta */
@@ -669,6 +668,7 @@ export class RawCoList<
     this.insertions = listAfter.insertions;
     this.lastValidTransaction = listAfter.lastValidTransaction;
     this.knownTransactions = listAfter.knownTransactions;
+    this.totalValidTransactions = listAfter.totalValidTransactions;
     this.deletionsByInsertion = listAfter.deletionsByInsertion;
     this._cachedEntries = undefined;
   }

package/src/coValues/coMap.ts

@@ -1,12 +1,11 @@
 import { CoID, RawCoValue } from "../coValue.js";
 import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
-import { AgentID, TransactionID } from "../ids.js";
+import { AgentID, RawCoID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
 import { RawAccountID } from "./account.js";
 import type { RawGroup } from "./group.js";
-import { Transaction } from "../coValueCore/verifiedState.js";
 
 type MapOp<K extends string, V extends JsonValue | undefined> = {
   txID: TransactionID;
@@ -61,7 +60,7 @@ export class RawCoMapView<
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
   };
   /** @internal */
-  knownTransactions: Set<Transaction>;
+  knownTransactions: Record<RawCoID, number>;
 
   /** @internal */
   ignorePrivateTransactions: boolean;
@@ -70,9 +69,7 @@ export class RawCoMapView<
   /** @category 6. Meta */
   readonly _shape!: Shape;
 
-  get totalValidTransactions() {
-    return this.knownTransactions.size;
-  }
+  totalValidTransactions: number = 0;
 
   /** @internal */
   constructor(
@@ -88,7 +85,10 @@ export class RawCoMapView<
       options?.ignorePrivateTransactions ?? false;
     this.ops = {};
     this.latest = {};
-    this.knownTransactions = new Set<Transaction>();
+
+    // We track the knownTransacions in multiple CoValues because branches
+    // need to retrieve the transactions from both the source and the branch
+    this.knownTransactions = { [core.id]: 0 };
 
     this.processNewTransactions();
   }
@@ -147,6 +147,8 @@ export class RawCoMapView<
     for (const [key, entries] of changedEntries.entries()) {
       this.latest[key] = entries[entries.length - 1];
     }
+
+    this.totalValidTransactions += newValidTransactions.length;
   }
 
   isTimeTravelEntity() {

package/src/coValues/coStream.ts

@@ -10,6 +10,7 @@ import { isCoValue } from "../typeUtils/isCoValue.js";
 import { RawAccountID } from "./account.js";
 import { RawGroup } from "./group.js";
 import { Transaction } from "../coValueCore/verifiedState.js";
+import { RawCoID } from "../ids.js";
 
 export type BinaryStreamInfo = {
   mimeType: string;
@@ -55,18 +56,16 @@ export class RawCoStreamView<
     [key: SessionID]: CoStreamItem<Item>[];
   };
   /** @internal */
-  knownTransactions: Set<Transaction>;
+  knownTransactions: Record<RawCoID, number>;
   readonly _item!: Item;
 
-  get totalValidTransactions() {
-    return this.knownTransactions.size;
-  }
+  totalValidTransactions: number = 0;
 
   constructor(core: AvailableCoValueCore) {
     this.id = core.id as CoID<this>;
     this.core = core;
     this.items = {};
-    this.knownTransactions = new Set<Transaction>();
+    this.knownTransactions = { [core.id]: 0 };
     this.processNewTransactions();
   }
 
@@ -128,6 +127,8 @@ export class RawCoStreamView<
     for (const entries of changeEntries) {
       entries.sort(this.compareStreamItems);
     }
+
+    this.totalValidTransactions += newValidTransactions.length;
   }
 
   getSingleStream(): Item[] | undefined {