cojson 0.18.6 → 0.18.7

package/src/coValueCore/coValueCore.ts

@@ -5,10 +5,10 @@ import type { RawCoValue } from "../coValue.js";
 import type { ControlledAccountOrAgent } from "../coValues/account.js";
 import type { RawGroup } from "../coValues/group.js";
 import { CO_VALUE_LOADING_CONFIG } from "../config.js";
+import { validateTxSizeLimitInBytes } from "../coValueContentMessage.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
-  Encrypted,
   Hash,
   KeyID,
   KeySecret,
@@ -17,7 +17,6 @@ import {
 } from "../crypto/crypto.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { parseJSON, safeParseJSON } from "../jsonStringify.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
 import { determineValidTransactions } from "../permissions.js";
@@ -29,10 +28,14 @@ import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 import { SessionMap } from "./SessionMap.js";
 import {
   MergeCommit,
+  BranchPointerCommit,
+  MergedTransactionMetadata,
   createBranch,
   getBranchId,
+  getBranchOwnerId,
   getBranchSource,
   mergeBranch,
+  BranchStartCommit,
 } from "./branching.js";
 import { type RawAccountID } from "../coValues/account.js";
 import { decodeTransactionChangesAndMeta } from "./decodeTransactionChangesAndMeta.js";
@@ -70,6 +73,9 @@ export type VerifiedTransaction = {
 
   // True if the meta information has been parsed and loaded in the CoValueCore
   hasMetaBeenParsed: boolean;
+
+  // The previous verified transaction for the same session
+  previous: VerifiedTransaction | undefined;
 };
 
 export type DecryptedTransaction = {
@@ -600,6 +606,7 @@ export class CoValueCore {
     changes: JsonValue[],
     privacy: "private" | "trusting",
     meta?: JsonObject,
+    madeAt?: number,
   ): boolean {
     if (!this.verified) {
       throw new Error(
@@ -607,6 +614,8 @@ export class CoValueCore {
       );
     }
 
+    validateTxSizeLimitInBytes(changes);
+
     // This is an ugly hack to get a unique but stable session ID for editing the current account
     const sessionID =
       this.verified.header.meta?.type === "account"
@@ -634,6 +643,7 @@ export class CoValueCore {
         keyID,
         keySecret,
         meta,
+        madeAt ?? Date.now(),
       );
     } else {
       result = this.verified.makeNewTrustingTransaction(
@@ -641,6 +651,7 @@ export class CoValueCore {
         signerAgent,
         changes,
         meta,
+        madeAt ?? Date.now(),
       );
     }
 
@@ -689,12 +700,13 @@ export class CoValueCore {
   }
 
   // The starting point of the branch, in case this CoValue is a branch
-  branchStart:
-    | { branch: CoValueKnownState["sessions"]; madeAt: number }
-    | undefined;
+  branchStart: { from: BranchStartCommit["from"]; madeAt: number } | undefined;
 
   // The list of merge commits that have been made
-  mergeCommits: { commit: MergeCommit; madeAt: number }[] = [];
+  mergeCommits: MergeCommit[] = [];
+  branches: BranchPointerCommit[] = [];
+  earliestTxMadeAt: number = Number.MAX_SAFE_INTEGER;
+  latestTxMadeAt: number = 0;
 
   // Reset the parsed transactions and branches, to validate them again from scratch when the group is updated
   resetParsedTransactions() {
@@ -710,6 +722,11 @@ export class CoValueCore {
   verifiedTransactions: VerifiedTransaction[] = [];
   private verifiedTransactionsKnownSessions: CoValueKnownState["sessions"] = {};
 
+  private lastVerifiedTransactionBySessionID: Record<
+    SessionID,
+    VerifiedTransaction
+  > = {};
+
   /**
    * Loads the new transaction from the SessionMap into verifiedTransactions as a VerifiedTransaction.
    *
@@ -752,7 +769,7 @@ export class CoValueCore {
               txIndex,
             };
 
-        this.verifiedTransactions.push({
+        const verifiedTransaction = {
           author: accountOrAgentIDfromSessionID(sessionID),
           txID,
           madeAt: tx.madeAt,
@@ -764,7 +781,20 @@ export class CoValueCore {
           hasInvalidMeta: false,
           hasMetaBeenParsed: false,
           tx,
-        });
+          previous: this.lastVerifiedTransactionBySessionID[sessionID],
+        };
+
+        if (verifiedTransaction.madeAt > this.latestTxMadeAt) {
+          this.latestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        if (verifiedTransaction.madeAt < this.earliestTxMadeAt) {
+          this.earliestTxMadeAt = verifiedTransaction.madeAt;
+        }
+
+        this.verifiedTransactions.push(verifiedTransaction);
+        this.lastVerifiedTransactionBySessionID[sessionID] =
+          verifiedTransaction;
       });
 
       this.verifiedTransactionsKnownSessions[sessionID] =
@@ -793,23 +823,54 @@ export class CoValueCore {
 
     transaction.hasMetaBeenParsed = true;
 
-    if (
-      transaction.meta?.["branch"] &&
-      (!this.branchStart || transaction.madeAt < this.branchStart.madeAt)
-    ) {
-      this.branchStart = {
-        branch: transaction.meta.branch as CoValueKnownState["sessions"],
-        madeAt: transaction.madeAt,
-      };
+    // Branch related meta information
+    if (this.isBranch()) {
+      // Check if the transaction is a branch start
+      if ("from" in transaction.meta) {
+        if (!this.branchStart || transaction.madeAt < this.branchStart.madeAt) {
+          const commit = transaction.meta as BranchStartCommit;
+
+          this.branchStart = {
+            from: commit.from,
+            madeAt: transaction.madeAt,
+          };
+        }
+      }
+
+      // Check if the transaction is a merged checkpoint for a branch
+      if ("merged" in transaction.meta) {
+        const mergeCommit = transaction.meta as MergeCommit;
+        this.mergeCommits.push(mergeCommit);
+      }
     }
 
-    if (transaction.meta?.["merge"]) {
-      const mergeCommit = transaction.meta as MergeCommit;
+    // Check if the transaction is a branch pointer
+    if ("branch" in transaction.meta) {
+      const branch = transaction.meta as BranchPointerCommit;
 
-      this.mergeCommits.push({
-        commit: mergeCommit,
-        madeAt: transaction.madeAt,
-      });
+      this.branches.push(branch);
+    }
+
+    // Check if the transaction has been merged from a branch
+    if ("mi" in transaction.meta) {
+      const meta = transaction.meta as MergedTransactionMetadata;
+
+      // Check if the transaction is a merge commit
+      const previousTransaction = transaction.previous?.txID;
+      const sessionID = meta.s ?? previousTransaction?.sessionID;
+
+      if (sessionID) {
+        transaction.txID = {
+          sessionID,
+          txIndex: meta.mi,
+          branch: meta.b ?? previousTransaction?.branch,
+        };
+      } else {
+        logger.error("Merge commit without session ID", {
+          txID: transaction.txID,
+          prev: previousTransaction ?? null,
+        });
+      }
     }
   }
 
@@ -876,7 +937,9 @@ export class CoValueCore {
       const { txID } = transaction;
 
       const from = options?.from?.[txID.sessionID] ?? -1;
-      const to = options?.to?.[txID.sessionID] ?? Infinity;
+
+      // Load the to filter index. Sessions that are not in the to filter will be skipped
+      const to = options?.to ? (options.to[txID.sessionID] ?? -1) : Infinity;
 
       // The txIndex starts at 0 and from/to are referring to the count of transactions
       if (from > txID.txIndex || to < txID.txIndex) {
@@ -889,7 +952,7 @@ export class CoValueCore {
     // If this is a branch, we load the valid transactions from the source
     if (source && this.branchStart && !options?.skipBranchSource) {
       const sourceTransactions = source.getValidTransactions({
-        to: this.branchStart.branch,
+        to: this.branchStart.from,
         ignorePrivateTransactions: options?.ignorePrivateTransactions ?? false,
         knownTransactions: options?.knownTransactions,
       });
@@ -915,15 +978,46 @@ export class CoValueCore {
   }
 
   getCurrentBranchName() {
-    return this.verified?.header.meta?.branch as string | undefined;
+    return this.verified?.branchName;
   }
 
   getCurrentBranchSourceId() {
-    return this.verified?.header.meta?.source as RawCoID | undefined;
+    return this.verified?.branchSourceId;
   }
 
   isBranch() {
-    return Boolean(this.getCurrentBranchSourceId());
+    return Boolean(this.verified?.branchSourceId);
+  }
+
+  hasBranch(name: string, ownerId?: RawCoID) {
+    // This function requires the meta information to be parsed, which might not be the case
+    // if the value content hasn't been loaded yet
+    this.parseNewTransactions(false);
+
+    const currentOwnerId = getBranchOwnerId(this);
+    return this.branches.some((item) => {
+      if (item.branch !== name) {
+        return false;
+      }
+
+      if (item.ownerId === ownerId) {
+        return true;
+      }
+
+      if (!ownerId) {
+        return item.ownerId === currentOwnerId;
+      }
+
+      if (!item.ownerId) {
+        return ownerId === currentOwnerId;
+      }
+    });
+  }
+
+  getMergeCommits() {
+    this.parseNewTransactions(false);
+
+    return this.mergeCommits;
   }
 
   getValidSortedTransactions(options?: {

package/src/coValueCore/verifiedState.ts

@@ -59,6 +59,8 @@ export class VerifiedState {
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
   private streamingKnownState?: CoValueKnownState["sessions"];
   public lastAccessed: number | undefined;
+  public branchSourceId?: RawCoID;
+  public branchName?: string;
 
   constructor(
     id: RawCoID,
@@ -74,6 +76,8 @@ export class VerifiedState {
     this.streamingKnownState = streamingKnownState
       ? { ...streamingKnownState }
       : undefined;
+    this.branchSourceId = header.meta?.source as RawCoID | undefined;
+    this.branchName = header.meta?.branch as string | undefined;
   }
 
   clone(): VerifiedState {
@@ -114,12 +118,14 @@ export class VerifiedState {
     signerAgent: ControlledAccountOrAgent,
     changes: JsonValue[],
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewTrustingTransaction(
       sessionID,
       signerAgent,
       changes,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;
@@ -135,6 +141,7 @@ export class VerifiedState {
     keyID: KeyID,
     keySecret: KeySecret,
     meta: JsonObject | undefined,
+    madeAt: number,
   ) {
     const result = this.sessions.makeNewPrivateTransaction(
       sessionID,
@@ -143,6 +150,7 @@ export class VerifiedState {
       keyID,
       keySecret,
       meta,
+      madeAt,
     );
 
     this._cachedNewContentSinceEmpty = undefined;

package/src/coValues/coList.ts

@@ -138,7 +138,13 @@ export class RawCoList<
       sessionEntry[opID.txIndex] = txEntry;
     }
 
+    // Check if the change index already exists, may be the case of double merges
+    if (txEntry[opID.changeIdx]) {
+      return false;
+    }
+
     txEntry[opID.changeIdx] = value;
+    return true;
   }
 
   private isDeleted(opID: OpID) {
@@ -216,13 +222,18 @@ export class RawCoList<
         };
 
         if (change.op === "pre" || change.op === "app") {
-          this.createInsertionsEntry(opID, {
+          const created = this.createInsertionsEntry(opID, {
             madeAt,
             predecessors: [],
             successors: [],
             change,
           });
 
+          // If the change index already exists, we don't need to process it again
+          if (!created) {
+            continue;
+          }
+
           if (change.op === "pre") {
             if (change.before === "end") {
               this.beforeEnd.push(opID);

package/src/coValues/coMap.ts

@@ -49,10 +49,17 @@ export class RawCoMapView<
   latest: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>;
   };
+
   /** @internal */
-  latestTxMadeAt: number;
+  get latestTxMadeAt(): number {
+    return this.core.latestTxMadeAt;
+  }
+
   /** @internal */
-  earliestTxMadeAt: number | null;
+  get earliestTxMadeAt(): number {
+    return this.core.earliestTxMadeAt;
+  }
+
   /** @internal */
   ops: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
@@ -80,8 +87,7 @@ export class RawCoMapView<
   ) {
     this.id = core.id as CoID<this>;
     this.core = core;
-    this.latestTxMadeAt = 0;
-    this.earliestTxMadeAt = null;
+
     this.ignorePrivateTransactions =
       options?.ignorePrivateTransactions ?? false;
     this.ops = {};
@@ -105,10 +111,6 @@ export class RawCoMapView<
       return;
     }
 
-    if (this.earliestTxMadeAt === null && newValidTransactions[0]) {
-      this.earliestTxMadeAt = newValidTransactions[0].madeAt;
-    }
-
     const { ops } = this;
 
     const changedEntries = new Map<
@@ -117,10 +119,6 @@ export class RawCoMapView<
     >();
 
     for (const { txID, changes, madeAt, tx } of newValidTransactions) {
-      if (madeAt > this.latestTxMadeAt) {
-        this.latestTxMadeAt = madeAt;
-      }
-
       for (let changeIdx = 0; changeIdx < changes.length; changeIdx++) {
         const change = changes[changeIdx] as MapOpPayload<
           keyof Shape & string,

package/src/coValues/group.ts

@@ -1070,6 +1070,9 @@ export class RawGroup<
 
     if (init) {
       map.assign(init, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      map.core.makeTransaction([], "trusting");
     }
 
     return map;
@@ -1101,6 +1104,9 @@ export class RawGroup<
 
     if (init?.length) {
       list.appendItems(init, undefined, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      list.core.makeTransaction([], "trusting");
     }
 
     return list;
@@ -1141,7 +1147,7 @@ export class RawGroup<
     meta?: C["headerMeta"],
     uniqueness: CoValueUniqueness = this.crypto.createdNowUnique(),
   ): C {
-    return this.core.node
+    const stream = this.core.node
       .createCoValue({
         type: "costream",
         ruleset: {
@@ -1152,6 +1158,13 @@ export class RawGroup<
         ...uniqueness,
       })
       .getCurrentContent() as C;
+
+    if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      stream.core.makeTransaction([], "trusting");
+    }
+
+    return stream;
   }
 
   /** @category 3. Value creation */

package/src/config.ts

@@ -7,12 +7,21 @@
 **/
 export const TRANSACTION_CONFIG = {
   MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+  /**
+   * Messages larger than this will be rejected when creating a transaction.
+   * The current limit is set at 1MB because that's the limit imposed by Cloudflare to Websocket messages.
+   */
+  MAX_TX_SIZE_BYTES: 1 * 1024 * 1024,
 };
 
 export function setMaxRecommendedTxSize(size: number) {
   TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
 }
 
+export function setMaxTxSizeBytes(size: number) {
+  TRANSACTION_CONFIG.MAX_TX_SIZE_BYTES = size;
+}
+
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
   TIMEOUT: 30_000,

package/src/crypto/PureJSCrypto.ts

@@ -32,6 +32,29 @@ import { ControlledAccountOrAgent } from "../coValues/account.js";
 
 type Blake3State = ReturnType<typeof blake3.create>;
 
+const x25519SharedSecretCache = new Map<string, Uint8Array>();
+
+function getx25519SharedSecret(
+  privateKeyA: SealerSecret,
+  publicKeyB: SealerID,
+): Uint8Array {
+  const cacheKey = `${privateKeyA}-${publicKeyB}`;
+  let sharedSecret = x25519SharedSecretCache.get(cacheKey);
+
+  if (!sharedSecret) {
+    const privateKeyABytes = base58.decode(
+      privateKeyA.substring("sealerSecret_z".length),
+    );
+    const publicKeyBBytes = base58.decode(
+      publicKeyB.substring("sealer_z".length),
+    );
+    sharedSecret = x25519.getSharedSecret(privateKeyABytes, publicKeyBBytes);
+    x25519SharedSecretCache.set(cacheKey, sharedSecret);
+  }
+
+  return sharedSecret;
+}
+
 /**
  * Pure JavaScript implementation of the CryptoProvider interface using noble-curves and noble-ciphers libraries.
  * This provides a fallback implementation that doesn't require WebAssembly, offering:
@@ -164,16 +187,10 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
     to: SealerID;
     nOnceMaterial: { in: RawCoID; tx: TransactionID };
   }): Sealed<T> {
+    const sharedSecret = getx25519SharedSecret(from, to);
     const nOnce = this.generateJsonNonce(nOnceMaterial);
-
-    const sealerPub = base58.decode(to.substring("sealer_z".length));
-
-    const senderPriv = base58.decode(from.substring("sealerSecret_z".length));
-
     const plaintext = textEncoder.encode(stableStringify(message));
 
-    const sharedSecret = x25519.getSharedSecret(senderPriv, sealerPub);
-
     const sealedBytes = xsalsa20poly1305(sharedSecret, nOnce).encrypt(
       plaintext,
     );
@@ -189,14 +206,9 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
   ): T | undefined {
     const nOnce = this.generateJsonNonce(nOnceMaterial);
 
-    const sealerPriv = base58.decode(sealer.substring("sealerSecret_z".length));
-
-    const senderPub = base58.decode(from.substring("sealer_z".length));
-
+    const sharedSecret = getx25519SharedSecret(sealer, from);
     const sealedBytes = base64URLtoBytes(sealed.substring("sealed_U".length));
 
-    const sharedSecret = x25519.getSharedSecret(sealerPriv, senderPub);
-
     const plaintext = xsalsa20poly1305(sharedSecret, nOnce).decrypt(
       sealedBytes,
     );

package/src/exports.ts

@@ -63,7 +63,12 @@ import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
-import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
+import {
+  DisconnectedError,
+  SyncManager,
+  emptyKnownState,
+  hwrServerPeerSelector,
+} from "./sync.js";
 
 import {
   getContentMessageSize,
@@ -163,6 +168,7 @@ export {
   LogLevel,
   base64URLtoBytes,
   bytesToBase64url,
+  hwrServerPeerSelector,
 };
 
 export type {

package/src/localNode.ts

@@ -479,8 +479,14 @@ export class LocalNode {
       return branch.getCurrentContent() as T;
     }
 
-    // Passing skipRetry to true because otherwise creating a new branch would always take 1 retry delay
-    await this.loadCoValueCore(branch.id, undefined, true);
+    // Do a synchronous check to see if the branch exists, if not we don't need to try to load the branch
+    if (!source.hasBranch(branchName, branchOwnerID)) {
+      return source
+        .createBranch(branchName, branchOwnerID)
+        .getCurrentContent() as T;
+    }
+
+    await this.loadCoValueCore(branch.id);
 
     if (!branch.isAvailable()) {
       return source

package/src/storage/storageAsync.ts

@@ -1,7 +1,6 @@
 import {
   createContentMessage,
   exceedsRecommendedSize,
-  getTransactionSize,
 } from "../coValueContentMessage.js";
 import {
   type CoValueCore,

package/src/sync.ts

@@ -219,28 +219,7 @@ export class SyncManager {
     }
   }
 
-  sendNewContentIncludingDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, true);
-  }
-
-  sendNewContentWithoutDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, false);
-  }
-
-  private sendNewContent(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-    includeDependencies: boolean,
-  ) {
+  sendNewContent(id: RawCoID, peer: PeerState, seen: Set<RawCoID> = new Set()) {
     if (seen.has(id)) {
       return;
     }
@@ -253,9 +232,10 @@ export class SyncManager {
       return;
     }
 
+    const includeDependencies = peer.role !== "server";
     if (includeDependencies) {
       for (const dependency of coValue.getDependedOnCoValues()) {
-        this.sendNewContentIncludingDependencies(dependency, peer, seen);
+        this.sendNewContent(dependency, peer, seen);
       }
     }
 
@@ -434,7 +414,7 @@ export class SyncManager {
     const coValue = this.local.getCoValue(msg.id);
 
     if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
+      this.sendNewContent(msg.id, peer);
       return;
     }
 
@@ -444,7 +424,7 @@ export class SyncManager {
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
+        this.sendNewContent(msg.id, peer);
         return;
       }
 
@@ -478,11 +458,7 @@ export class SyncManager {
     }
 
     if (coValue.isAvailable()) {
-      if (peer.role === "server") {
-        this.sendNewContentWithoutDependencies(msg.id, peer);
-      } else {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
-      }
+      this.sendNewContent(msg.id, peer);
     }
   }
 
@@ -777,7 +753,7 @@ export class SyncManager {
 
       // We directly forward the new content to peers that have an active subscription
       if (peer.optimisticKnownStates.has(coValue.id)) {
-        this.sendNewContentIncludingDependencies(coValue.id, peer);
+        this.sendNewContent(coValue.id, peer);
         syncedPeers.push(peer);
       } else if (
         peer.role === "server" &&
@@ -808,7 +784,7 @@ export class SyncManager {
   handleCorrection(msg: KnownStateMessage, peer: PeerState) {
     peer.setKnownState(msg.id, knownStateIn(msg));
 
-    return this.sendNewContentIncludingDependencies(msg.id, peer);
+    return this.sendNewContent(msg.id, peer);
   }
 
   private syncQueue = new LocalTransactionsSyncQueue((content) =>