cojson 0.18.6 → 0.18.8

package/src/coValueCore/branching.ts

@@ -1,9 +1,41 @@
-import type { CoValueCore, JsonValue } from "../exports.js";
-import type { RawCoID, SessionID, TransactionID } from "../ids.js";
+import type { CoValueCore } from "../exports.js";
+import type { RawCoID, SessionID } from "../ids.js";
 import { type AvailableCoValueCore, idforHeader } from "./coValueCore.js";
 import type { CoValueHeader } from "./verifiedState.js";
 import type { CoValueKnownState } from "../sync.js";
-import type { ListOpPayload, OpID } from "../coValues/coList.js";
+
+/**
+ * Commit to identify the starting point of the branch
+ *
+ * In case of clonflicts, the first commit of this kind is considered the source of truth
+ */
+export type BranchStartCommit = {
+  from: CoValueKnownState["sessions"];
+};
+
+/**
+ * Commit that tracks a branch creation
+ */
+export type BranchPointerCommit = {
+  branch: string;
+  ownerId?: RawCoID;
+};
+
+/**
+ * Meta information attached to each merged transaction to retrieve the original transaction ID
+ */
+export type MergedTransactionMetadata = {
+  mi: number; // Transaction index and marker of a merge commit
+  s?: SessionID;
+  b?: RawCoID;
+};
+
+/**
+ * Merge commit located in a branch to track how many transactions have already been merged
+ */
+export type MergeCommit = {
+  merged: CoValueKnownState["sessions"];
+};
 
 export function getBranchHeader({
   type,
@@ -49,30 +81,37 @@ export function getBranchId(
     );
   }
 
-  if (!ownerId) {
-    const header = coValue.verified.header;
-
-    // Group and account coValues can't have branches, so we return the source id
-    if (header.ruleset.type !== "ownedByGroup") {
-      return coValue.id;
-    }
+  const currentOwnerId = ownerId ?? getBranchOwnerId(coValue);
 
-    ownerId = header.ruleset.group;
+  if (!currentOwnerId) {
+    return coValue.id;
   }
 
   const header = getBranchHeader({
     type: coValue.verified.header.type,
     branchName: name,
-    ownerId,
+    ownerId: currentOwnerId,
     sourceId: coValue.id,
   });
 
   return idforHeader(header, coValue.node.crypto);
 }
 
-export type BranchCommit = {
-  branch: CoValueKnownState["sessions"];
-};
+export function getBranchOwnerId(coValue: CoValueCore) {
+  if (!coValue.verified) {
+    throw new Error(
+      "CoValueCore: getBranchOwnerId called on coValue without verified state",
+    );
+  }
+
+  const header = coValue.verified.header;
+
+  if (header.ruleset.type !== "ownedByGroup") {
+    return undefined;
+  }
+
+  return header.ruleset.group;
+}
 
 /**
  * Given a coValue, a branch name and an owner id, creates a new branch CoValue
@@ -88,32 +127,34 @@ export function createBranch(
     );
   }
 
-  if (!ownerId) {
-    const header = coValue.verified.header;
-
-    // Group and account coValues can't have branches, so we return the source coValue
-    if (header.ruleset.type !== "ownedByGroup") {
-      return coValue;
-    }
+  const branchOwnerId = ownerId ?? getBranchOwnerId(coValue);
 
-    ownerId = header.ruleset.group;
+  if (!branchOwnerId) {
+    return coValue;
   }
 
   const header = getBranchHeader({
     type: coValue.verified.header.type,
     branchName: name,
-    ownerId,
+    ownerId: branchOwnerId,
     sourceId: coValue.id,
   });
 
-  const value = coValue.node.createCoValue(header);
+  const branch = coValue.node.createCoValue(header);
+  const sessions = { ...coValue.knownState().sessions };
 
   // Create a branch commit to identify the starting point of the branch
-  value.makeTransaction([], "private", {
-    branch: coValue.knownState().sessions,
-  } satisfies BranchCommit);
+  branch.makeTransaction([], "private", {
+    from: sessions,
+  } satisfies BranchStartCommit);
+
+  // Create a branch pointer, to identify that we created a branch
+  coValue.makeTransaction([], "private", {
+    branch: name,
+    ownerId,
+  } satisfies BranchPointerCommit);
 
-  return value;
+  return branch;
 }
 
 /**
@@ -141,15 +182,6 @@ export function getBranchSource(
   return source;
 }
 
-export type MergeCommit = {
-  // The point where the branch was merged
-  merge: CoValueKnownState["sessions"];
-  // The id of the branch that was merged
-  id: RawCoID;
-  // The number of transactions that were merged, will be used in the future to handle the edits history properly
-  count: number;
-};
-
 /**
  * Given a branch coValue, merges the branch into the source coValue
  */
@@ -164,12 +196,6 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
     return branch;
   }
 
-  const sourceId = branch.getCurrentBranchSourceId();
-
-  if (!sourceId) {
-    throw new Error("CoValueCore: mergeBranch called on a non-branch coValue");
-  }
-
   const target = getBranchSource(branch);
 
   if (!target) {
@@ -178,13 +204,9 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
 
   // Look for previous merge commits, to see which transactions needs to be merged
   // Done mostly for performance reasons, as we could merge all the transactions every time and nothing would change
-  const mergedTransactions = target.mergeCommits.reduce(
-    (acc, { commit }) => {
-      if (commit.id !== branch.id) {
-        return acc;
-      }
-
-      for (const [sessionID, count] of Object.entries(commit.merge) as [
+  const mergedTransactions = branch.getMergeCommits().reduce(
+    (acc, { merged }) => {
+      for (const [sessionID, count] of Object.entries(merged) as [
         SessionID,
         number,
       ][]) {
@@ -210,111 +232,34 @@ export function mergeBranch(branch: CoValueCore): CoValueCore {
     return target;
   }
 
-  // Create a merge commit to identify the merge point
-  target.makeTransaction([], "private", {
-    merge: { ...branch.knownState().sessions },
-    id: branch.id,
-    count: branchValidTransactions.length,
-  } satisfies MergeCommit);
-
-  const currentSessionID = target.node.currentSessionID;
+  // We do track in the meta information the original txID to make sure that
+  // the CoList opid still point to the correct transaction
+  // To reduce the cost of the meta we skip the repeated information
+  let lastSessionId: string | undefined = undefined;
+  let lastBranchId: string | undefined = undefined;
 
-  if (
-    target.verified.header.type === "colist" ||
-    target.verified.header.type === "coplaintext"
-  ) {
-    const mapping: Record<`${SessionID}:${number}`, number> = {};
+  for (const tx of branchValidTransactions) {
+    const mergeMeta: MergedTransactionMetadata = {
+      mi: tx.txID.txIndex,
+    };
 
-    const session = target.verified.sessions.get(currentSessionID);
-    let txIdx = session ? session.transactions.length : 0;
-
-    // Create a mapping from the branch transactions to the target transactions
-    for (const { txID } of branchValidTransactions) {
-      mapping[`${txID.sessionID}:${txID.txIndex}`] = txIdx;
-      txIdx++;
+    if (lastSessionId !== tx.txID.sessionID) {
+      mergeMeta.s = tx.txID.sessionID;
     }
 
-    for (const { tx, changes } of branchValidTransactions) {
-      target.makeTransaction(
-        mapCoListChangesToTarget(
-          changes as ListOpPayload<JsonValue>[],
-          currentSessionID,
-          mapping,
-        ),
-        tx.privacy,
-      );
-    }
-  } else {
-    for (const { tx, changes } of branchValidTransactions) {
-      target.makeTransaction(changes, tx.privacy);
+    if (lastBranchId !== tx.txID.branch) {
+      mergeMeta.b = tx.txID.branch;
     }
-  }
 
-  return target;
-}
-
-/**
- * Given a list of changes, maps the opIDs to the target transactions
- */
-function mapCoListChangesToTarget(
-  changes: ListOpPayload<JsonValue>[],
-  currentSessionID: SessionID,
-  mapping: Record<`${SessionID}:${number}`, number>,
-) {
-  return changes.map((change) => {
-    if (change.op === "app") {
-      if (change.after === "start") {
-        return change;
-      }
-
-      return {
-        ...change,
-        after: convertOpID(change.after, currentSessionID, mapping),
-      };
-    }
-
-    if (change.op === "del") {
-      return {
-        ...change,
-        insertion: convertOpID(change.insertion, currentSessionID, mapping),
-      };
-    }
-
-    if (change.op === "pre") {
-      if (change.before === "end") {
-        return change;
-      }
-
-      return {
-        ...change,
-        before: convertOpID(change.before, currentSessionID, mapping),
-      };
-    }
-
-    return change;
-  });
-}
-
-function convertOpID(
-  opID: OpID,
-  sessionID: SessionID,
-  mapping: Record<`${SessionID}:${number}`, number>,
-) {
-  // If the opID comes from the source branch, we don't need to map it
-  if (!opID.branch) {
-    return opID;
+    target.makeTransaction(tx.changes, tx.tx.privacy, mergeMeta, tx.madeAt);
+    lastSessionId = tx.txID.sessionID;
+    lastBranchId = tx.txID.branch;
   }
 
-  const mappedIndex = mapping[`${opID.sessionID}:${opID.txIndex}`];
-
-  // If the opID doesn't exist in the mapping, we don't need to map it
-  if (mappedIndex === undefined) {
-    return opID;
-  }
+  // Track the merged transactions for the branch, so future merges will know which transactions have already been merged
+  branch.makeTransaction([], "private", {
+    merged: branch.knownState().sessions,
+  } satisfies MergeCommit);
 
-  return {
-    sessionID: sessionID,
-    txIndex: mappedIndex,
-    changeIdx: opID.changeIdx,
-  };
+  return target;
 }

package/src/coValueCore/coValueCore.ts

@@ -5,10 +5,10 @@ import type { RawCoValue } from "../coValue.js";
 import type { ControlledAccountOrAgent } from "../coValues/account.js";
 import type { RawGroup } from "../coValues/group.js";
 import { CO_VALUE_LOADING_CONFIG } from "../config.js";
+import { validateTxSizeLimitInBytes } from "../coValueContentMessage.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
-  Encrypted,
   Hash,
   KeyID,
   KeySecret,
@@ -17,7 +17,6 @@ import {
 } from "../crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { parseJSON, safeParseJSON } from "../jsonStringify.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
 import { determineValidTransactions } from "../permissions.js";
@@ -29,10 +28,14 @@ import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 import { SessionMap } from "./SessionMap.js";
 import {
   MergeCommit,
+  BranchPointerCommit,
+  MergedTransactionMetadata,
   createBranch,
   getBranchId,
+  getBranchOwnerId,
   getBranchSource,
   mergeBranch,
+  BranchStartCommit,
 } from "./branching.js";
 import { type RawAccountID } from "../coValues/account.js";
 import { decodeTransactionChangesAndMeta } from "./decodeTransactionChangesAndMeta.js";
@@ -70,6 +73,9 @@ export type VerifiedTransaction = {
 
   // True if the meta information has been parsed and loaded in the CoValueCore
   hasMetaBeenParsed: boolean;
+
+  // The previous verified transaction for the same session
+  previous: VerifiedTransaction | undefined;
 };
 
 export type DecryptedTransaction = {
@@ -600,6 +606,7 @@ export class CoValueCore {
     changes: JsonValue[],
     privacy: "private" | "trusting",
     meta?: JsonObject,
+    madeAt?: number,
   ): boolean {
     if (!this.verified) {
       throw new Error(
@@ -607,6 +614,8 @@ export class CoValueCore {
       );
     }
 
+    validateTxSizeLimitInBytes(changes);
+
     // This is an ugly hack to get a unique but stable session ID for editing the current account
     const sessionID =
       this.verified.header.meta?.type === "account"
@@ -634,6 +643,7 @@ export class CoValueCore {
         keyID,
         keySecret,
         meta,
+        madeAt ?? Date.now(),
       );
     } else {
       result = this.verified.makeNewTrustingTransaction(
@@ -641,6 +651,7 @@ export class CoValueCore {
         signerAgent,
         changes,
         meta,
+        madeAt ?? Date.now(),
       );
     }
 
@@ -689,12 +700,13 @@ export class CoValueCore {
   }
 
   // The starting point of the branch, in case this CoValue is a branch
-  branchStart:
-    | { branch: CoValueKnownState["sessions"]; madeAt: number }
-    | undefined;
+  branchStart: { from: BranchStartCommit["from"]; madeAt: number } | undefined;
 
   // The list of merge commits that have been made
-  mergeCommits: { commit: MergeCommit; madeAt: number }[] = [];
+  mergeCommits: MergeCommit[] = [];
+  branches: BranchPointerCommit[] = [];
+  earliestTxMadeAt: number = Number.MAX_SAFE_INTEGER;
+  latestTxMadeAt: number = 0;
 
   // Reset the parsed transactions and branches, to validate them again from scratch when the group is updated
   resetParsedTransactions() {
@@ -710,6 +722,11 @@ export class CoValueCore {
   verifiedTransactions: VerifiedTransaction[] = [];
   private verifiedTransactionsKnownSessions: CoValueKnownState["sessions"] = {};
 
+  private lastVerifiedTransactionBySessionID: Record<
+    SessionID,
+    VerifiedTransaction
+  > = {};
+
   /**
    * Loads the new transaction from the SessionMap into verifiedTransactions as a VerifiedTransaction.
    *
@@ -752,7 +769,7 @@ export class CoValueCore {
               txIndex,
             };
 
-        this.verifiedTransactions.push({
+        const verifiedTransaction = {
           author: accountOrAgentIDfromSessionID(sessionID),
           txID,
           madeAt: tx.madeAt,
@@ -764,7 +781,20 @@ export class CoValueCore {
           hasInvalidMeta: false,
           hasMetaBeenParsed: false,
           tx,
-        });
+          previous: this.lastVerifiedTransactionBySessionID[sessionID],
+        };
+
+        if (verifiedTransaction.madeAt > this.latestTxMadeAt) {
+          this.latestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        if (verifiedTransaction.madeAt < this.earliestTxMadeAt) {
+          this.earliestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        this.verifiedTransactions.push(verifiedTransaction);
+        this.lastVerifiedTransactionBySessionID[sessionID] =
+          verifiedTransaction;
       });
 
       this.verifiedTransactionsKnownSessions[sessionID] =
@@ -793,23 +823,54 @@ export class CoValueCore {
 
     transaction.hasMetaBeenParsed = true;
 
-    if (
-      transaction.meta?.["branch"] &&
-      (!this.branchStart || transaction.madeAt < this.branchStart.madeAt)
-    ) {
-      this.branchStart = {
-        branch: transaction.meta.branch as CoValueKnownState["sessions"],
-        madeAt: transaction.madeAt,
-      };
+    // Branch related meta information
+    if (this.isBranch()) {
+      // Check if the transaction is a branch start
+      if ("from" in transaction.meta) {
+        if (!this.branchStart || transaction.madeAt < this.branchStart.madeAt) {
+          const commit = transaction.meta as BranchStartCommit;
+
+          this.branchStart = {
+            from: commit.from,
+            madeAt: transaction.madeAt,
+          };
+        }
+      }
+
+      // Check if the transaction is a merged checkpoint for a branch
+      if ("merged" in transaction.meta) {
+        const mergeCommit = transaction.meta as MergeCommit;
+        this.mergeCommits.push(mergeCommit);
+      }
     }
 
-    if (transaction.meta?.["merge"]) {
-      const mergeCommit = transaction.meta as MergeCommit;
+    // Check if the transaction is a branch pointer
+    if ("branch" in transaction.meta) {
+      const branch = transaction.meta as BranchPointerCommit;
 
-      this.mergeCommits.push({
-        commit: mergeCommit,
-        madeAt: transaction.madeAt,
-      });
+      this.branches.push(branch);
+    }
+
+    // Check if the transaction has been merged from a branch
+    if ("mi" in transaction.meta) {
+      const meta = transaction.meta as MergedTransactionMetadata;
+
+      // Check if the transaction is a merge commit
+      const previousTransaction = transaction.previous?.txID;
+      const sessionID = meta.s ?? previousTransaction?.sessionID;
+
+      if (sessionID) {
+        transaction.txID = {
+          sessionID,
+          txIndex: meta.mi,
+          branch: meta.b ?? previousTransaction?.branch,
+        };
+      } else {
+        logger.error("Merge commit without session ID", {
+          txID: transaction.txID,
+          prev: previousTransaction ?? null,
+        });
+      }
     }
   }
 
@@ -876,7 +937,9 @@ export class CoValueCore {
       const { txID } = transaction;
 
       const from = options?.from?.[txID.sessionID] ?? -1;
-      const to = options?.to?.[txID.sessionID] ?? Infinity;
+
+      // Load the to filter index. Sessions that are not in the to filter will be skipped
+      const to = options?.to ? (options.to[txID.sessionID] ?? -1) : Infinity;
 
       // The txIndex starts at 0 and from/to are referring to the count of transactions
       if (from > txID.txIndex || to < txID.txIndex) {
@@ -889,7 +952,7 @@ export class CoValueCore {
     // If this is a branch, we load the valid transactions from the source
     if (source && this.branchStart && !options?.skipBranchSource) {
       const sourceTransactions = source.getValidTransactions({
-        to: this.branchStart.branch,
+        to: this.branchStart.from,
         ignorePrivateTransactions: options?.ignorePrivateTransactions ?? false,
         knownTransactions: options?.knownTransactions,
       });
@@ -915,15 +978,46 @@ export class CoValueCore {
   }
 
   getCurrentBranchName() {
-    return this.verified?.header.meta?.branch as string | undefined;
+    return this.verified?.branchName;
   }
 
   getCurrentBranchSourceId() {
-    return this.verified?.header.meta?.source as RawCoID | undefined;
+    return this.verified?.branchSourceId;
   }
 
   isBranch() {
-    return Boolean(this.getCurrentBranchSourceId());
+    return Boolean(this.verified?.branchSourceId);
+  }
+
+  hasBranch(name: string, ownerId?: RawCoID) {
+    // This function requires the meta information to be parsed, which might not be the case
+    // if the value content hasn't been loaded yet
+    this.parseNewTransactions(false);
+
+    const currentOwnerId = getBranchOwnerId(this);
+    return this.branches.some((item) => {
+      if (item.branch !== name) {
+        return false;
+      }
+
+      if (item.ownerId === ownerId) {
+        return true;
+      }
+
+      if (!ownerId) {
+        return item.ownerId === currentOwnerId;
+      }
+
+      if (!item.ownerId) {
+        return ownerId === currentOwnerId;
+      }
+    });
+  }
+
+  getMergeCommits() {
+    this.parseNewTransactions(false);
+
+    return this.mergeCommits;
   }
 
   getValidSortedTransactions(options?: {

package/src/coValueCore/verifiedState.ts

@@ -59,6 +59,8 @@ export class VerifiedState {
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   private streamingKnownState?: CoValueKnownState["sessions"];
   public lastAccessed: number | undefined;
+  public branchSourceId?: RawCoID;
+  public branchName?: string;
 
   constructor(
     id: RawCoID,
@@ -74,6 +76,8 @@ export class VerifiedState {
     this.streamingKnownState = streamingKnownState
       ? { ...streamingKnownState }
       : undefined;
+    this.branchSourceId = header.meta?.source as RawCoID | undefined;
+    this.branchName = header.meta?.branch as string | undefined;
   }
 
   clone(): VerifiedState {
@@ -114,12 +118,14 @@ export class VerifiedState {
     signerAgent: ControlledAccountOrAgent,
     changes: JsonValue[],
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewTrustingTransaction(
       sessionID,
       signerAgent,
       changes,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;
@@ -135,6 +141,7 @@ export class VerifiedState {
     keyID: KeyID,
     keySecret: KeySecret,
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewPrivateTransaction(
       sessionID,
@@ -143,6 +150,7 @@ export class VerifiedState {
       keyID,
       keySecret,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;

package/src/coValues/coList.ts

@@ -138,7 +138,13 @@ export class RawCoList<
       sessionEntry[opID.txIndex] = txEntry;
     }
 
+    // Check if the change index already exists, may be the case of double merges
+    if (txEntry[opID.changeIdx]) {
+      return false;
+    }
+
     txEntry[opID.changeIdx] = value;
+    return true;
   }
 
   private isDeleted(opID: OpID) {
@@ -216,13 +222,18 @@ export class RawCoList<
         };
 
         if (change.op === "pre" || change.op === "app") {
-          this.createInsertionsEntry(opID, {
+          const created = this.createInsertionsEntry(opID, {
             madeAt,
             predecessors: [],
             successors: [],
             change,
           });
 
+          // If the change index already exists, we don't need to process it again
+          if (!created) {
+            continue;
+          }
+
           if (change.op === "pre") {
             if (change.before === "end") {
               this.beforeEnd.push(opID);