cojson 0.18.5 → 0.18.7

package/src/coValues/coList.ts

@@ -1,11 +1,7 @@
 import { CoID, RawCoValue } from "../coValue.js";
-import {
-  AvailableCoValueCore,
-  CoValueCore,
-} from "../coValueCore/coValueCore.js";
+import { AvailableCoValueCore } from "../coValueCore/coValueCore.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
-import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
 import { RawAccountID } from "./account.js";
@@ -39,12 +35,14 @@ type InsertionEntry<T extends JsonValue> = {
   madeAt: number;
   predecessors: OpID[];
   successors: OpID[];
-} & InsertionOpPayload<T>;
+  change: InsertionOpPayload<T>;
+};
 
 type DeletionEntry = {
   madeAt: number;
   deletionID: OpID;
-} & DeletionOpPayload;
+  change: DeletionOpPayload;
+};
 
 export class RawCoList<
   Item extends JsonValue = JsonValue,
@@ -109,6 +107,89 @@ export class RawCoList<
     this.processNewTransactions();
   }
 
+  private getInsertionsEntry(opID: OpID) {
+    const index = getSessionIndex(opID);
+
+    const sessionEntry = this.insertions[index];
+    if (!sessionEntry) {
+      return undefined;
+    }
+
+    const txEntry = sessionEntry[opID.txIndex];
+    if (!txEntry) {
+      return undefined;
+    }
+
+    return txEntry[opID.changeIdx];
+  }
+
+  private createInsertionsEntry(opID: OpID, value: InsertionEntry<Item>) {
+    const index = getSessionIndex(opID);
+
+    let sessionEntry = this.insertions[index];
+    if (!sessionEntry) {
+      sessionEntry = {};
+      this.insertions[index] = sessionEntry;
+    }
+
+    let txEntry = sessionEntry[opID.txIndex];
+    if (!txEntry) {
+      txEntry = {};
+      sessionEntry[opID.txIndex] = txEntry;
+    }
+
+    // Check if the change index already exists, may be the case of double merges
+    if (txEntry[opID.changeIdx]) {
+      return false;
+    }
+
+    txEntry[opID.changeIdx] = value;
+    return true;
+  }
+
+  private isDeleted(opID: OpID) {
+    const index = getSessionIndex(opID);
+
+    const sessionEntry = this.deletionsByInsertion[index];
+
+    if (!sessionEntry) {
+      return false;
+    }
+
+    const txEntry = sessionEntry[opID.txIndex];
+
+    if (!txEntry) {
+      return false;
+    }
+
+    return Boolean(txEntry[opID.changeIdx]?.length);
+  }
+
+  private pushDeletionsByInsertionEntry(opID: OpID, value: DeletionEntry) {
+    const index = getSessionIndex(opID);
+
+    let sessionEntry = this.deletionsByInsertion[index];
+    if (!sessionEntry) {
+      sessionEntry = {};
+      this.deletionsByInsertion[index] = sessionEntry;
+    }
+
+    let txEntry = sessionEntry[opID.txIndex];
+    if (!txEntry) {
+      txEntry = {};
+      sessionEntry[opID.txIndex] = txEntry;
+    }
+
+    let list = txEntry[opID.changeIdx];
+
+    if (!list) {
+      list = [];
+      txEntry[opID.changeIdx] = list;
+    }
+
+    list.push(value);
+  }
+
   processNewTransactions() {
     const transactions = this.core.getValidSortedTransactions({
       ignorePrivateTransactions: false,
@@ -133,87 +214,56 @@ export class RawCoList<
       for (const [changeIdx, changeUntyped] of changes.entries()) {
         const change = changeUntyped as ListOpPayload<Item>;
 
+        const opID = {
+          sessionID: txID.sessionID,
+          txIndex: txID.txIndex,
+          branch: txID.branch,
+          changeIdx,
+        };
+
         if (change.op === "pre" || change.op === "app") {
-          let sessionEntry = this.insertions[txID.sessionID];
-          if (!sessionEntry) {
-            sessionEntry = {};
-            this.insertions[txID.sessionID] = sessionEntry;
-          }
-          let txEntry = sessionEntry[txID.txIndex];
-          if (!txEntry) {
-            txEntry = {};
-            sessionEntry[txID.txIndex] = txEntry;
-          }
-          txEntry[changeIdx] = {
+          const created = this.createInsertionsEntry(opID, {
             madeAt,
             predecessors: [],
             successors: [],
-            ...change,
-          };
+            change,
+          });
+
+          // If the change index already exists, we don't need to process it again
+          if (!created) {
+            continue;
+          }
+
           if (change.op === "pre") {
             if (change.before === "end") {
-              this.beforeEnd.push({
-                ...txID,
-                changeIdx,
-              });
+              this.beforeEnd.push(opID);
             } else {
-              const beforeEntry =
-                this.insertions[change.before.sessionID]?.[
-                  change.before.txIndex
-                ]?.[change.before.changeIdx];
+              const beforeEntry = this.getInsertionsEntry(change.before);
+
               if (!beforeEntry) {
                 continue;
               }
-              beforeEntry.predecessors.splice(0, 0, {
-                ...txID,
-                changeIdx,
-              });
+
+              beforeEntry.predecessors.push(opID);
             }
           } else {
             if (change.after === "start") {
-              this.afterStart.push({
-                ...txID,
-                changeIdx,
-              });
+              this.afterStart.push(opID);
             } else {
-              const afterEntry =
-                this.insertions[change.after.sessionID]?.[
-                  change.after.txIndex
-                ]?.[change.after.changeIdx];
+              const afterEntry = this.getInsertionsEntry(change.after);
+
               if (!afterEntry) {
                 continue;
               }
-              afterEntry.successors.push({
-                ...txID,
-                changeIdx,
-              });
+
+              afterEntry.successors.push(opID);
             }
           }
         } else if (change.op === "del") {
-          let sessionEntry =
-            this.deletionsByInsertion[change.insertion.sessionID];
-          if (!sessionEntry) {
-            sessionEntry = {};
-            this.deletionsByInsertion[change.insertion.sessionID] =
-              sessionEntry;
-          }
-          let txEntry = sessionEntry[change.insertion.txIndex];
-          if (!txEntry) {
-            txEntry = {};
-            sessionEntry[change.insertion.txIndex] = txEntry;
-          }
-          let changeEntry = txEntry[change.insertion.changeIdx];
-          if (!changeEntry) {
-            changeEntry = [];
-            txEntry[change.insertion.changeIdx] = changeEntry;
-          }
-          changeEntry.push({
+          this.pushDeletionsByInsertionEntry(change.insertion, {
             madeAt,
-            deletionID: {
-              ...txID,
-              changeIdx,
-            },
-            ...change,
+            deletionID: opID,
+            change,
           });
         } else {
           throw new Error(
@@ -324,10 +374,7 @@ export class RawCoList<
     while (todo.length > 0) {
       const currentOpID = todo[todo.length - 1]!;
 
-      const entry =
-        this.insertions[currentOpID.sessionID]?.[currentOpID.txIndex]?.[
-          currentOpID.changeIdx
-        ];
+      const entry = this.getInsertionsEntry(currentOpID);
 
       if (!entry) {
         throw new Error("Missing op " + currentOpID);
@@ -338,22 +385,19 @@ export class RawCoList<
 
       // We navigate the predecessors before processing the current opID in the list
       if (shouldTraversePredecessors) {
-        for (let i = entry.predecessors.length - 1; i >= 0; i--) {
-          todo.push(entry.predecessors[i]!);
+        for (const predecessor of entry.predecessors) {
+          todo.push(predecessor);
         }
         predecessorsVisited.add(currentOpID);
       } else {
         // Remove the current opID from the todo stack to consider it processed.
         todo.pop();
 
-        const deleted =
-          (this.deletionsByInsertion[currentOpID.sessionID]?.[
-            currentOpID.txIndex
-          ]?.[currentOpID.changeIdx]?.length || 0) > 0;
+        const deleted = this.isDeleted(currentOpID);
 
         if (!deleted) {
           arr.push({
-            value: entry.value,
+            value: entry.change.value,
             madeAt: entry.madeAt,
             opID: currentOpID,
           });
@@ -629,3 +673,10 @@ export class RawCoList<
     this._cachedEntries = undefined;
   }
 }
+
+function getSessionIndex(txID: TransactionID): SessionID {
+  if (txID.branch) {
+    return `${txID.sessionID}_branch_${txID.branch}`;
+  }
+  return txID.sessionID;
+}

package/src/coValues/coMap.ts

@@ -49,10 +49,17 @@ export class RawCoMapView<
   latest: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>;
   };
+
   /** @internal */
-  latestTxMadeAt: number;
+  get latestTxMadeAt(): number {
+    return this.core.latestTxMadeAt;
+  }
+
   /** @internal */
-  earliestTxMadeAt: number | null;
+  get earliestTxMadeAt(): number {
+    return this.core.earliestTxMadeAt;
+  }
+
   /** @internal */
   ops: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
@@ -80,8 +87,7 @@ export class RawCoMapView<
   ) {
     this.id = core.id as CoID<this>;
     this.core = core;
-    this.latestTxMadeAt = 0;
-    this.earliestTxMadeAt = null;
+
     this.ignorePrivateTransactions =
       options?.ignorePrivateTransactions ?? false;
     this.ops = {};
@@ -105,10 +111,6 @@ export class RawCoMapView<
       return;
     }
 
-    if (this.earliestTxMadeAt === null && newValidTransactions[0]) {
-      this.earliestTxMadeAt = newValidTransactions[0].madeAt;
-    }
-
     const { ops } = this;
 
     const changedEntries = new Map<
@@ -117,10 +119,6 @@ export class RawCoMapView<
     >();
 
     for (const { txID, changes, madeAt, tx } of newValidTransactions) {
-      if (madeAt > this.latestTxMadeAt) {
-        this.latestTxMadeAt = madeAt;
-      }
-
       for (let changeIdx = 0; changeIdx < changes.length; changeIdx++) {
         const change = changes[changeIdx] as MapOpPayload<
           keyof Shape & string,

package/src/coValues/group.ts

@@ -1070,6 +1070,9 @@ export class RawGroup<
 
     if (init) {
       map.assign(init, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      map.core.makeTransaction([], "trusting");
     }
 
     return map;
@@ -1101,6 +1104,9 @@ export class RawGroup<
 
     if (init?.length) {
       list.appendItems(init, undefined, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      list.core.makeTransaction([], "trusting");
     }
 
     return list;
@@ -1141,7 +1147,7 @@ export class RawGroup<
     meta?: C["headerMeta"],
     uniqueness: CoValueUniqueness = this.crypto.createdNowUnique(),
   ): C {
-    return this.core.node
+    const stream = this.core.node
       .createCoValue({
         type: "costream",
         ruleset: {
@@ -1152,6 +1158,13 @@ export class RawGroup<
         ...uniqueness,
       })
       .getCurrentContent() as C;
+
+    if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      stream.core.makeTransaction([], "trusting");
+    }
+
+    return stream;
   }
 
   /** @category 3. Value creation */

package/src/config.ts

@@ -7,12 +7,21 @@
 **/
 export const TRANSACTION_CONFIG = {
   MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+  /**
+   * Messages larger than this will be rejected when creating a transaction.
+   * The current limit is set at 1MB because that's the limit imposed by Cloudflare to Websocket messages.
+   */
+  MAX_TX_SIZE_BYTES: 1 * 1024 * 1024,
 };
 
 export function setMaxRecommendedTxSize(size: number) {
   TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
 }
 
+export function setMaxTxSizeBytes(size: number) {
+  TRANSACTION_CONFIG.MAX_TX_SIZE_BYTES = size;
+}
+
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
   TIMEOUT: 30_000,

package/src/crypto/PureJSCrypto.ts

@@ -32,6 +32,29 @@ import { ControlledAccountOrAgent } from "../coValues/account.js";
 
 type Blake3State = ReturnType<typeof blake3.create>;
 
+const x25519SharedSecretCache = new Map<string, Uint8Array>();
+
+function getx25519SharedSecret(
+  privateKeyA: SealerSecret,
+  publicKeyB: SealerID,
+): Uint8Array {
+  const cacheKey = `${privateKeyA}-${publicKeyB}`;
+  let sharedSecret = x25519SharedSecretCache.get(cacheKey);
+
+  if (!sharedSecret) {
+    const privateKeyABytes = base58.decode(
+      privateKeyA.substring("sealerSecret_z".length),
+    );
+    const publicKeyBBytes = base58.decode(
+      publicKeyB.substring("sealer_z".length),
+    );
+    sharedSecret = x25519.getSharedSecret(privateKeyABytes, publicKeyBBytes);
+    x25519SharedSecretCache.set(cacheKey, sharedSecret);
+  }
+
+  return sharedSecret;
+}
+
 /**
  * Pure JavaScript implementation of the CryptoProvider interface using noble-curves and noble-ciphers libraries.
  * This provides a fallback implementation that doesn't require WebAssembly, offering:
@@ -164,16 +187,10 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
     to: SealerID;
     nOnceMaterial: { in: RawCoID; tx: TransactionID };
   }): Sealed<T> {
+    const sharedSecret = getx25519SharedSecret(from, to);
     const nOnce = this.generateJsonNonce(nOnceMaterial);
-
-    const sealerPub = base58.decode(to.substring("sealer_z".length));
-
-    const senderPriv = base58.decode(from.substring("sealerSecret_z".length));
-
     const plaintext = textEncoder.encode(stableStringify(message));
 
-    const sharedSecret = x25519.getSharedSecret(senderPriv, sealerPub);
-
     const sealedBytes = xsalsa20poly1305(sharedSecret, nOnce).encrypt(
       plaintext,
     );
@@ -189,14 +206,9 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
   ): T | undefined {
     const nOnce = this.generateJsonNonce(nOnceMaterial);
 
-    const sealerPriv = base58.decode(sealer.substring("sealerSecret_z".length));
-
-    const senderPub = base58.decode(from.substring("sealer_z".length));
-
+    const sharedSecret = getx25519SharedSecret(sealer, from);
     const sealedBytes = base64URLtoBytes(sealed.substring("sealed_U".length));
 
-    const sharedSecret = x25519.getSharedSecret(sealerPriv, senderPub);
-
     const plaintext = xsalsa20poly1305(sharedSecret, nOnce).decrypt(
       sealedBytes,
     );

package/src/exports.ts

@@ -1,6 +1,10 @@
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 import { type RawCoValue } from "./coValue.js";
-import { CoValueCore, idforHeader } from "./coValueCore/coValueCore.js";
+import {
+  CoValueCore,
+  idforHeader,
+  type AvailableCoValueCore,
+} from "./coValueCore/coValueCore.js";
 import { CoValueUniqueness } from "./coValueCore/verifiedState.js";
 import {
   ControlledAccount,
@@ -59,7 +63,12 @@ import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
-import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
+import {
+  DisconnectedError,
+  SyncManager,
+  emptyKnownState,
+  hwrServerPeerSelector,
+} from "./sync.js";
 
 import {
   getContentMessageSize,
@@ -159,6 +168,7 @@ export {
   LogLevel,
   base64URLtoBytes,
   bytesToBase64url,
+  hwrServerPeerSelector,
 };
 
 export type {
@@ -171,6 +181,7 @@ export type {
   BinaryStreamStart,
   OpID,
   AccountRole,
+  AvailableCoValueCore,
 };
 
 export * from "./storage/index.js";

package/src/ids.ts

@@ -23,6 +23,7 @@ export function rawCoIDfromBytes(bytes: Uint8Array): RawCoID {
 export type TransactionID = {
   sessionID: SessionID;
   txIndex: number;
+  branch?: RawCoID;
 };
 
 export type AgentID = `sealer_z${string}/signer_z${string}`;

package/src/localNode.ts

@@ -479,8 +479,14 @@ export class LocalNode {
       return branch.getCurrentContent() as T;
     }
 
-    // Passing skipRetry to true because otherwise creating a new branch would always take 1 retry delay
-    await this.loadCoValueCore(branch.id, undefined, true);
+    // Do a synchronous check to see if the branch exists, if not we don't need to try to load the branch
+    if (!source.hasBranch(branchName, branchOwnerID)) {
+      return source
+        .createBranch(branchName, branchOwnerID)
+        .getCurrentContent() as T;
+    }
+
+    await this.loadCoValueCore(branch.id);
 
     if (!branch.isAvailable()) {
       return source

package/src/storage/storageAsync.ts

@@ -1,7 +1,6 @@
 import {
   createContentMessage,
   exceedsRecommendedSize,
-  getTransactionSize,
 } from "../coValueContentMessage.js";
 import {
   type CoValueCore,
@@ -110,10 +109,14 @@ export class StorageApiAsync implements StorageAPI {
 
       let idx = 0;
 
-      signatures.push({
-        idx: sessionRow.lastIdx,
-        signature: sessionRow.lastSignature,
-      });
+      const lastSignature = signatures[signatures.length - 1];
+
+      if (lastSignature?.signature !== sessionRow.lastSignature) {
+        signatures.push({
+          idx: sessionRow.lastIdx,
+          signature: sessionRow.lastSignature,
+        });
+      }
 
       for (const signature of signatures) {
         const newTxsInSession = await this.dbClient.getNewTransactionInSession(

package/src/storage/storageSync.ts

@@ -113,10 +113,14 @@ export class StorageApiSync implements StorageAPI {
 
       let idx = 0;
 
-      signatures.push({
-        idx: sessionRow.lastIdx,
-        signature: sessionRow.lastSignature,
-      });
+      const lastSignature = signatures[signatures.length - 1];
+
+      if (lastSignature?.signature !== sessionRow.lastSignature) {
+        signatures.push({
+          idx: sessionRow.lastIdx,
+          signature: sessionRow.lastSignature,
+        });
+      }
 
       for (const signature of signatures) {
         const newTxsInSession = this.dbClient.getNewTransactionInSession(

package/src/sync.ts

@@ -219,28 +219,7 @@ export class SyncManager {
     }
   }
 
-  sendNewContentIncludingDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, true);
-  }
-
-  sendNewContentWithoutDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, false);
-  }
-
-  private sendNewContent(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-    includeDependencies: boolean,
-  ) {
+  sendNewContent(id: RawCoID, peer: PeerState, seen: Set<RawCoID> = new Set()) {
     if (seen.has(id)) {
       return;
     }
@@ -253,9 +232,10 @@ export class SyncManager {
       return;
     }
 
+    const includeDependencies = peer.role !== "server";
     if (includeDependencies) {
       for (const dependency of coValue.getDependedOnCoValues()) {
-        this.sendNewContentIncludingDependencies(dependency, peer, seen);
+        this.sendNewContent(dependency, peer, seen);
       }
     }
 
@@ -434,7 +414,7 @@ export class SyncManager {
     const coValue = this.local.getCoValue(msg.id);
 
     if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
+      this.sendNewContent(msg.id, peer);
       return;
     }
 
@@ -444,7 +424,7 @@ export class SyncManager {
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
+        this.sendNewContent(msg.id, peer);
         return;
       }
 
@@ -478,11 +458,7 @@ export class SyncManager {
     }
 
     if (coValue.isAvailable()) {
-      if (peer.role === "server") {
-        this.sendNewContentWithoutDependencies(msg.id, peer);
-      } else {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
-      }
+      this.sendNewContent(msg.id, peer);
     }
   }
 
@@ -777,7 +753,7 @@ export class SyncManager {
 
       // We directly forward the new content to peers that have an active subscription
       if (peer.optimisticKnownStates.has(coValue.id)) {
-        this.sendNewContentIncludingDependencies(coValue.id, peer);
+        this.sendNewContent(coValue.id, peer);
         syncedPeers.push(peer);
       } else if (
         peer.role === "server" &&
@@ -808,7 +784,7 @@ export class SyncManager {
   handleCorrection(msg: KnownStateMessage, peer: PeerState) {
     peer.setKnownState(msg.id, knownStateIn(msg));
 
-    return this.sendNewContentIncludingDependencies(msg.id, peer);
+    return this.sendNewContent(msg.id, peer);
   }
 
   private syncQueue = new LocalTransactionsSyncQueue((content) =>