cojson 0.18.5 → 0.18.7

package/src/coValueCore/branching.ts

@@ -1,8 +1,41 @@
-import { CoValueCore } from "../exports.js";
-import { RawCoID, SessionID } from "../ids.js";
-import { AvailableCoValueCore, idforHeader } from "./coValueCore.js";
-import { CoValueHeader } from "./verifiedState.js";
-import { CoValueKnownState } from "../sync.js";
+import type { CoValueCore } from "../exports.js";
+import type { RawCoID, SessionID } from "../ids.js";
+import { type AvailableCoValueCore, idforHeader } from "./coValueCore.js";
+import type { CoValueHeader } from "./verifiedState.js";
+import type { CoValueKnownState } from "../sync.js";
+
+/**
+ * Commit to identify the starting point of the branch
+ *
+ * In case of clonflicts, the first commit of this kind is considered the source of truth
+ */
+export type BranchStartCommit = {
+  from: CoValueKnownState["sessions"];
+};
+
+/**
+ * Commit that tracks a branch creation
+ */
+export type BranchPointerCommit = {
+  branch: string;
+  ownerId?: RawCoID;
+};
+
+/**
+ * Meta information attached to each merged transaction to retrieve the original transaction ID
+ */
+export type MergedTransactionMetadata = {
+  mi: number; // Transaction index and marker of a merge commit
+  s?: SessionID;
+  b?: RawCoID;
+};
+
+/**
+ * Merge commit located in a branch to track how many transactions have already been merged
+ */
+export type MergeCommit = {
+  merged: CoValueKnownState["sessions"];
+};
 
 export function getBranchHeader({
   type,
@@ -48,30 +81,37 @@ export function getBranchId(
     );
   }
 
-  if (!ownerId) {
-    const header = coValue.verified.header;
+  const currentOwnerId = ownerId ?? getBranchOwnerId(coValue);
 
-    // Group and account coValues can't have branches, so we return the source id
-    if (header.ruleset.type !== "ownedByGroup") {
-      return coValue.id;
-    }
-
-    ownerId = header.ruleset.group;
+  if (!currentOwnerId) {
+    return coValue.id;
   }
 
   const header = getBranchHeader({
     type: coValue.verified.header.type,
     branchName: name,
-    ownerId,
+    ownerId: currentOwnerId,
     sourceId: coValue.id,
   });
 
   return idforHeader(header, coValue.node.crypto);
 }
 
-export type BranchCommit = {
-  branch: CoValueKnownState["sessions"];
-};
+export function getBranchOwnerId(coValue: CoValueCore) {
+  if (!coValue.verified) {
+    throw new Error(
+      "CoValueCore: getBranchOwnerId called on coValue without verified state",
+    );
+  }
+
+  const header = coValue.verified.header;
+
+  if (header.ruleset.type !== "ownedByGroup") {
+    return undefined;
+  }
+
+  return header.ruleset.group;
+}
 
 /**
  * Given a coValue, a branch name and an owner id, creates a new branch CoValue
@@ -87,32 +127,34 @@ export function createBranch(
     );
   }
 
-  if (!ownerId) {
-    const header = coValue.verified.header;
-
-    // Group and account coValues can't have branches, so we return the source coValue
-    if (header.ruleset.type !== "ownedByGroup") {
-      return coValue;
-    }
+  const branchOwnerId = ownerId ?? getBranchOwnerId(coValue);
 
-    ownerId = header.ruleset.group;
+  if (!branchOwnerId) {
+    return coValue;
   }
 
   const header = getBranchHeader({
     type: coValue.verified.header.type,
     branchName: name,
-    ownerId,
+    ownerId: branchOwnerId,
     sourceId: coValue.id,
   });
 
-  const value = coValue.node.createCoValue(header);
+  const branch = coValue.node.createCoValue(header);
+  const sessions = { ...coValue.knownState().sessions };
 
   // Create a branch commit to identify the starting point of the branch
-  value.makeTransaction([], "private", {
-    branch: coValue.knownState().sessions,
-  } satisfies BranchCommit);
+  branch.makeTransaction([], "private", {
+    from: sessions,
+  } satisfies BranchStartCommit);
+
+  // Create a branch pointer, to identify that we created a branch
+  coValue.makeTransaction([], "private", {
+    branch: name,
+    ownerId,
+  } satisfies BranchPointerCommit);
 
-  return value;
+  return branch;
 }
 
 /**
@@ -125,7 +167,7 @@ export function getBranchSource(
     return undefined;
   }
 
-  const sourceId = coValue.verified.header.meta?.source;
+  const sourceId = coValue.getCurrentBranchSourceId();
 
   if (!sourceId) {
     return undefined;
@@ -140,15 +182,6 @@ export function getBranchSource(
   return source;
 }
 
-export type MergeCommit = {
-  // The point where the branch was merged
-  merge: CoValueKnownState["sessions"];
-  // The id of the branch that was merged
-  id: RawCoID;
-  // The number of transactions that were merged, will be used in the future to handle the edits history properly
-  count: number;
-};
-
 /**
  * Given a branch coValue, merges the branch into the source coValue
  */
@@ -159,10 +192,8 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
     );
   }
 
-  const sourceId = branch.verified.header.meta?.source;
-
-  if (!sourceId) {
-    throw new Error("CoValueCore: mergeBranch called on a non-branch coValue");
+  if (branch.verified.header.ruleset.type !== "ownedByGroup") {
+    return branch;
   }
 
   const target = getBranchSource(branch);
@@ -173,13 +204,9 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
 
   // Look for previous merge commits, to see which transactions needs to be merged
   // Done mostly for performance reasons, as we could merge all the transactions every time and nothing would change
-  const mergedTransactions = target.mergeCommits.reduce(
-    (acc, { commit }) => {
-      if (commit.id !== branch.id) {
-        return acc;
-      }
-
-      for (const [sessionID, count] of Object.entries(commit.merge) as [
+  const mergedTransactions = branch.getMergeCommits().reduce(
+    (acc, { merged }) => {
+      for (const [sessionID, count] of Object.entries(merged) as [
         SessionID,
         number,
       ][]) {
@@ -205,16 +232,34 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
     return target;
   }
 
-  // Create a merge commit to identify the merge point
-  target.makeTransaction([], "private", {
-    merge: { ...branch.knownState().sessions },
-    id: branch.id,
-    count: branchValidTransactions.length,
-  } satisfies MergeCommit);
+  // We do track in the meta information the original txID to make sure that
+  // the CoList opid still point to the correct transaction
+  // To reduce the cost of the meta we skip the repeated information
+  let lastSessionId: string | undefined = undefined;
+  let lastBranchId: string | undefined = undefined;
+
+  for (const tx of branchValidTransactions) {
+    const mergeMeta: MergedTransactionMetadata = {
+      mi: tx.txID.txIndex,
+    };
 
-  for (const { tx, changes } of branchValidTransactions) {
-    target.makeTransaction(changes, tx.privacy);
+    if (lastSessionId !== tx.txID.sessionID) {
+      mergeMeta.s = tx.txID.sessionID;
+    }
+
+    if (lastBranchId !== tx.txID.branch) {
+      mergeMeta.b = tx.txID.branch;
+    }
+
+    target.makeTransaction(tx.changes, tx.tx.privacy, mergeMeta, tx.madeAt);
+    lastSessionId = tx.txID.sessionID;
+    lastBranchId = tx.txID.branch;
   }
 
+  // Track the merged transactions for the branch, so future merges will know which transactions have already been merged
+  branch.makeTransaction([], "private", {
+    merged: branch.knownState().sessions,
+  } satisfies MergeCommit);
+
   return target;
 }

package/src/coValueCore/coValueCore.ts

@@ -5,10 +5,10 @@ import type { RawCoValue } from "../coValue.js";
 import type { ControlledAccountOrAgent } from "../coValues/account.js";
 import type { RawGroup } from "../coValues/group.js";
 import { CO_VALUE_LOADING_CONFIG } from "../config.js";
+import { validateTxSizeLimitInBytes } from "../coValueContentMessage.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
-  Encrypted,
   Hash,
   KeyID,
   KeySecret,
@@ -17,7 +17,6 @@ import {
 } from "../crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { parseJSON, safeParseJSON } from "../jsonStringify.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
 import { determineValidTransactions } from "../permissions.js";
@@ -29,10 +28,14 @@ import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 import { SessionMap } from "./SessionMap.js";
 import {
   MergeCommit,
+  BranchPointerCommit,
+  MergedTransactionMetadata,
   createBranch,
   getBranchId,
+  getBranchOwnerId,
   getBranchSource,
   mergeBranch,
+  BranchStartCommit,
 } from "./branching.js";
 import { type RawAccountID } from "../coValues/account.js";
 import { decodeTransactionChangesAndMeta } from "./decodeTransactionChangesAndMeta.js";
@@ -70,6 +73,9 @@ export type VerifiedTransaction = {
 
   // True if the meta information has been parsed and loaded in the CoValueCore
   hasMetaBeenParsed: boolean;
+
+  // The previous verified transaction for the same session
+  previous: VerifiedTransaction | undefined;
 };
 
 export type DecryptedTransaction = {
@@ -215,6 +221,20 @@ export class CoValueCore {
     });
   }
 
+  waitForFullStreaming(): Promise<CoValueCore> {
+    return new Promise<CoValueCore>((resolve) => {
+      const listener = (core: CoValueCore) => {
+        if (core.isAvailable() && !core.verified.isStreaming()) {
+          resolve(core);
+          this.listeners.delete(listener);
+        }
+      };
+
+      this.listeners.add(listener);
+      listener(this);
+    });
+  }
+
   getStateForPeer(peerId: PeerID) {
     return this.peers.get(peerId);
   }
@@ -586,6 +606,7 @@ export class CoValueCore {
     changes: JsonValue[],
     privacy: "private" | "trusting",
     meta?: JsonObject,
+    madeAt?: number,
   ): boolean {
     if (!this.verified) {
       throw new Error(
@@ -593,6 +614,8 @@ export class CoValueCore {
       );
     }
 
+    validateTxSizeLimitInBytes(changes);
+
     // This is an ugly hack to get a unique but stable session ID for editing the current account
     const sessionID =
       this.verified.header.meta?.type === "account"
@@ -620,6 +643,7 @@ export class CoValueCore {
         keyID,
         keySecret,
         meta,
+        madeAt ?? Date.now(),
       );
     } else {
       result = this.verified.makeNewTrustingTransaction(
@@ -627,6 +651,7 @@ export class CoValueCore {
         signerAgent,
         changes,
         meta,
+        madeAt ?? Date.now(),
       );
     }
 
@@ -675,12 +700,13 @@ export class CoValueCore {
   }
 
   // The starting point of the branch, in case this CoValue is a branch
-  branchStart:
-    | { branch: CoValueKnownState["sessions"]; madeAt: number }
-    | undefined;
+  branchStart: { from: BranchStartCommit["from"]; madeAt: number } | undefined;
 
   // The list of merge commits that have been made
-  mergeCommits: { commit: MergeCommit; madeAt: number }[] = [];
+  mergeCommits: MergeCommit[] = [];
+  branches: BranchPointerCommit[] = [];
+  earliestTxMadeAt: number = Number.MAX_SAFE_INTEGER;
+  latestTxMadeAt: number = 0;
 
   // Reset the parsed transactions and branches, to validate them again from scratch when the group is updated
   resetParsedTransactions() {
@@ -696,6 +722,11 @@ export class CoValueCore {
   verifiedTransactions: VerifiedTransaction[] = [];
   private verifiedTransactionsKnownSessions: CoValueKnownState["sessions"] = {};
 
+  private lastVerifiedTransactionBySessionID: Record<
+    SessionID,
+    VerifiedTransaction
+  > = {};
+
   /**
    * Loads the new transaction from the SessionMap into verifiedTransactions as a VerifiedTransaction.
    *
@@ -717,6 +748,8 @@ export class CoValueCore {
       return;
     }
 
+    const isBranch = this.isBranch();
+
     for (const [sessionID, sessionLog] of this.verified.sessions.entries()) {
       const count = this.verifiedTransactionsKnownSessions[sessionID] ?? 0;
 
@@ -725,12 +758,20 @@ export class CoValueCore {
           return;
         }
 
-        this.verifiedTransactions.push({
+        const txID = isBranch
+          ? {
+              sessionID,
+              txIndex,
+              branch: this.id,
+            }
+          : {
+              sessionID,
+              txIndex,
+            };
+
+        const verifiedTransaction = {
           author: accountOrAgentIDfromSessionID(sessionID),
-          txID: {
-            sessionID,
-            txIndex,
-          },
+          txID,
           madeAt: tx.madeAt,
           isValidated: false,
           isValid: false,
@@ -740,7 +781,20 @@ export class CoValueCore {
           hasInvalidMeta: false,
           hasMetaBeenParsed: false,
           tx,
-        });
+          previous: this.lastVerifiedTransactionBySessionID[sessionID],
+        };
+
+        if (verifiedTransaction.madeAt > this.latestTxMadeAt) {
+          this.latestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        if (verifiedTransaction.madeAt < this.earliestTxMadeAt) {
+          this.earliestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        this.verifiedTransactions.push(verifiedTransaction);
+        this.lastVerifiedTransactionBySessionID[sessionID] =
+          verifiedTransaction;
       });
 
       this.verifiedTransactionsKnownSessions[sessionID] =
@@ -769,23 +823,54 @@ export class CoValueCore {
 
     transaction.hasMetaBeenParsed = true;
 
-    if (
-      transaction.meta?.["branch"] &&
-      (!this.branchStart || transaction.madeAt < this.branchStart.madeAt)
-    ) {
-      this.branchStart = {
-        branch: transaction.meta.branch as CoValueKnownState["sessions"],
-        madeAt: transaction.madeAt,
-      };
+    // Branch related meta information
+    if (this.isBranch()) {
+      // Check if the transaction is a branch start
+      if ("from" in transaction.meta) {
+        if (!this.branchStart || transaction.madeAt < this.branchStart.madeAt) {
+          const commit = transaction.meta as BranchStartCommit;
+
+          this.branchStart = {
+            from: commit.from,
+            madeAt: transaction.madeAt,
+          };
+        }
+      }
+
+      // Check if the transaction is a merged checkpoint for a branch
+      if ("merged" in transaction.meta) {
+        const mergeCommit = transaction.meta as MergeCommit;
+        this.mergeCommits.push(mergeCommit);
+      }
     }
 
-    if (transaction.meta?.["merge"]) {
-      const mergeCommit = transaction.meta as MergeCommit;
+    // Check if the transaction is a branch pointer
+    if ("branch" in transaction.meta) {
+      const branch = transaction.meta as BranchPointerCommit;
 
-      this.mergeCommits.push({
-        commit: mergeCommit,
-        madeAt: transaction.madeAt,
-      });
+      this.branches.push(branch);
+    }
+
+    // Check if the transaction has been merged from a branch
+    if ("mi" in transaction.meta) {
+      const meta = transaction.meta as MergedTransactionMetadata;
+
+      // Check if the transaction is a merge commit
+      const previousTransaction = transaction.previous?.txID;
+      const sessionID = meta.s ?? previousTransaction?.sessionID;
+
+      if (sessionID) {
+        transaction.txID = {
+          sessionID,
+          txIndex: meta.mi,
+          branch: meta.b ?? previousTransaction?.branch,
+        };
+      } else {
+        logger.error("Merge commit without session ID", {
+          txID: transaction.txID,
+          prev: previousTransaction ?? null,
+        });
+      }
     }
   }
 
@@ -836,6 +921,8 @@ export class CoValueCore {
 
     const matchingTransactions: DecryptedTransaction[] = [];
 
+    const source = getBranchSource(this);
+
     for (const transaction of this.verifiedTransactions) {
       if (!isValidTransactionWithChanges(transaction)) {
         continue;
@@ -847,10 +934,12 @@ export class CoValueCore {
 
       options?.knownTransactions?.add(transaction.tx);
 
-      const { txID, madeAt } = transaction;
+      const { txID } = transaction;
 
       const from = options?.from?.[txID.sessionID] ?? -1;
-      const to = options?.to?.[txID.sessionID] ?? Infinity;
+
+      // Load the to filter index. Sessions that are not in the to filter will be skipped
+      const to = options?.to ? (options.to[txID.sessionID] ?? -1) : Infinity;
 
       // The txIndex starts at 0 and from/to are referring to the count of transactions
       if (from > txID.txIndex || to < txID.txIndex) {
@@ -860,26 +949,16 @@ export class CoValueCore {
       matchingTransactions.push(transaction);
     }
 
-    const source = getBranchSource(this);
-
     // If this is a branch, we load the valid transactions from the source
     if (source && this.branchStart && !options?.skipBranchSource) {
       const sourceTransactions = source.getValidTransactions({
-        to: this.branchStart.branch,
+        to: this.branchStart.from,
         ignorePrivateTransactions: options?.ignorePrivateTransactions ?? false,
         knownTransactions: options?.knownTransactions,
       });
 
-      for (const { changes, tx, madeAt, txID } of sourceTransactions) {
-        matchingTransactions.push({
-          txID: {
-            sessionID: `${txID.sessionID}_branch_${source.id}`,
-            txIndex: txID.txIndex,
-          },
-          madeAt,
-          changes,
-          tx,
-        });
+      for (const transaction of sourceTransactions) {
+        matchingTransactions.push(transaction);
       }
     }
 
@@ -898,6 +977,49 @@ export class CoValueCore {
     return this.node.getCoValue(getBranchId(this, name, ownerId));
   }
 
+  getCurrentBranchName() {
+    return this.verified?.branchName;
+  }
+
+  getCurrentBranchSourceId() {
+    return this.verified?.branchSourceId;
+  }
+
+  isBranch() {
+    return Boolean(this.verified?.branchSourceId);
+  }
+
+  hasBranch(name: string, ownerId?: RawCoID) {
+    // This function requires the meta information to be parsed, which might not be the case
+    // if the value content hasn't been loaded yet
+    this.parseNewTransactions(false);
+
+    const currentOwnerId = getBranchOwnerId(this);
+    return this.branches.some((item) => {
+      if (item.branch !== name) {
+        return false;
+      }
+
+      if (item.ownerId === ownerId) {
+        return true;
+      }
+
+      if (!ownerId) {
+        return item.ownerId === currentOwnerId;
+      }
+
+      if (!item.ownerId) {
+        return ownerId === currentOwnerId;
+      }
+    });
+  }
+
+  getMergeCommits() {
+    this.parseNewTransactions(false);
+
+    return this.mergeCommits;
+  }
+
   getValidSortedTransactions(options?: {
     ignorePrivateTransactions: boolean;
 

package/src/coValueCore/verifiedState.ts

@@ -59,6 +59,8 @@ export class VerifiedState {
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   private streamingKnownState?: CoValueKnownState["sessions"];
   public lastAccessed: number | undefined;
+  public branchSourceId?: RawCoID;
+  public branchName?: string;
 
   constructor(
     id: RawCoID,
@@ -74,6 +76,8 @@ export class VerifiedState {
     this.streamingKnownState = streamingKnownState
       ? { ...streamingKnownState }
       : undefined;
+    this.branchSourceId = header.meta?.source as RawCoID | undefined;
+    this.branchName = header.meta?.branch as string | undefined;
   }
 
   clone(): VerifiedState {
@@ -114,12 +118,14 @@ export class VerifiedState {
     signerAgent: ControlledAccountOrAgent,
     changes: JsonValue[],
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewTrustingTransaction(
       sessionID,
       signerAgent,
       changes,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;
@@ -135,6 +141,7 @@ export class VerifiedState {
     keyID: KeyID,
     keySecret: KeySecret,
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewPrivateTransaction(
       sessionID,
@@ -143,6 +150,7 @@ export class VerifiedState {
       keyID,
       keySecret,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;