cojson 0.18.6 → 0.18.8

package/src/coValues/coMap.ts

@@ -49,10 +49,17 @@ export class RawCoMapView<
   latest: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>;
   };
+
   /** @internal */
-  latestTxMadeAt: number;
+  get latestTxMadeAt(): number {
+    return this.core.latestTxMadeAt;
+  }
+
   /** @internal */
-  earliestTxMadeAt: number | null;
+  get earliestTxMadeAt(): number {
+    return this.core.earliestTxMadeAt;
+  }
+
   /** @internal */
   ops: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>[];
@@ -80,8 +87,7 @@ export class RawCoMapView<
   ) {
     this.id = core.id as CoID<this>;
     this.core = core;
-    this.latestTxMadeAt = 0;
-    this.earliestTxMadeAt = null;
+
     this.ignorePrivateTransactions =
       options?.ignorePrivateTransactions ?? false;
     this.ops = {};
@@ -105,10 +111,6 @@ export class RawCoMapView<
       return;
     }
 
-    if (this.earliestTxMadeAt === null && newValidTransactions[0]) {
-      this.earliestTxMadeAt = newValidTransactions[0].madeAt;
-    }
-
     const { ops } = this;
 
     const changedEntries = new Map<
@@ -117,10 +119,6 @@ export class RawCoMapView<
     >();
 
     for (const { txID, changes, madeAt, tx } of newValidTransactions) {
-      if (madeAt > this.latestTxMadeAt) {
-        this.latestTxMadeAt = madeAt;
-      }
-
       for (let changeIdx = 0; changeIdx < changes.length; changeIdx++) {
         const change = changes[changeIdx] as MapOpPayload<
           keyof Shape & string,

package/src/coValues/group.ts

@@ -1070,6 +1070,9 @@ export class RawGroup<
 
     if (init) {
       map.assign(init, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      map.core.makeTransaction([], "trusting");
     }
 
     return map;
@@ -1101,6 +1104,9 @@ export class RawGroup<
 
     if (init?.length) {
       list.appendItems(init, undefined, initPrivacy);
+    } else if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      list.core.makeTransaction([], "trusting");
     }
 
     return list;
@@ -1141,7 +1147,7 @@ export class RawGroup<
     meta?: C["headerMeta"],
     uniqueness: CoValueUniqueness = this.crypto.createdNowUnique(),
   ): C {
-    return this.core.node
+    const stream = this.core.node
       .createCoValue({
         type: "costream",
         ruleset: {
@@ -1152,6 +1158,13 @@ export class RawGroup<
         ...uniqueness,
       })
       .getCurrentContent() as C;
+
+    if (!uniqueness.createdAt) {
+      // If the createdAt is not set, we need to make a trusting transaction to set the createdAt
+      stream.core.makeTransaction([], "trusting");
+    }
+
+    return stream;
   }
 
   /** @category 3. Value creation */

package/src/config.ts

@@ -7,12 +7,21 @@
 **/
 export const TRANSACTION_CONFIG = {
   MAX_RECOMMENDED_TX_SIZE: 100 * 1024,
+  /**
+   * Messages larger than this will be rejected when creating a transaction.
+   * The current limit is set at 1MB because that's the limit imposed by Cloudflare to Websocket messages.
+   */
+  MAX_TX_SIZE_BYTES: 1 * 1024 * 1024,
 };
 
 export function setMaxRecommendedTxSize(size: number) {
   TRANSACTION_CONFIG.MAX_RECOMMENDED_TX_SIZE = size;
 }
 
+export function setMaxTxSizeBytes(size: number) {
+  TRANSACTION_CONFIG.MAX_TX_SIZE_BYTES = size;
+}
+
 export const CO_VALUE_LOADING_CONFIG = {
   MAX_RETRIES: 1,
   TIMEOUT: 30_000,

package/src/crypto/PureJSCrypto.ts

@@ -32,6 +32,29 @@ import { ControlledAccountOrAgent } from "../coValues/account.js";
 
 type Blake3State = ReturnType<typeof blake3.create>;
 
+const x25519SharedSecretCache = new Map<string, Uint8Array>();
+
+function getx25519SharedSecret(
+  privateKeyA: SealerSecret,
+  publicKeyB: SealerID,
+): Uint8Array {
+  const cacheKey = `${privateKeyA}-${publicKeyB}`;
+  let sharedSecret = x25519SharedSecretCache.get(cacheKey);
+
+  if (!sharedSecret) {
+    const privateKeyABytes = base58.decode(
+      privateKeyA.substring("sealerSecret_z".length),
+    );
+    const publicKeyBBytes = base58.decode(
+      publicKeyB.substring("sealer_z".length),
+    );
+    sharedSecret = x25519.getSharedSecret(privateKeyABytes, publicKeyBBytes);
+    x25519SharedSecretCache.set(cacheKey, sharedSecret);
+  }
+
+  return sharedSecret;
+}
+
 /**
  * Pure JavaScript implementation of the CryptoProvider interface using noble-curves and noble-ciphers libraries.
  * This provides a fallback implementation that doesn't require WebAssembly, offering:
@@ -164,16 +187,10 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
     to: SealerID;
     nOnceMaterial: { in: RawCoID; tx: TransactionID };
   }): Sealed<T> {
+    const sharedSecret = getx25519SharedSecret(from, to);
     const nOnce = this.generateJsonNonce(nOnceMaterial);
-
-    const sealerPub = base58.decode(to.substring("sealer_z".length));
-
-    const senderPriv = base58.decode(from.substring("sealerSecret_z".length));
-
     const plaintext = textEncoder.encode(stableStringify(message));
 
-    const sharedSecret = x25519.getSharedSecret(senderPriv, sealerPub);
-
     const sealedBytes = xsalsa20poly1305(sharedSecret, nOnce).encrypt(
       plaintext,
     );
@@ -189,14 +206,9 @@ export class PureJSCrypto extends CryptoProvider<Blake3State> {
   ): T | undefined {
     const nOnce = this.generateJsonNonce(nOnceMaterial);
 
-    const sealerPriv = base58.decode(sealer.substring("sealerSecret_z".length));
-
-    const senderPub = base58.decode(from.substring("sealer_z".length));
-
+    const sharedSecret = getx25519SharedSecret(sealer, from);
     const sealedBytes = base64URLtoBytes(sealed.substring("sealed_U".length));
 
-    const sharedSecret = x25519.getSharedSecret(sealerPriv, senderPub);
-
     const plaintext = xsalsa20poly1305(sharedSecret, nOnce).decrypt(
       sealedBytes,
     );

package/src/exports.ts

@@ -63,7 +63,12 @@ import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { disablePermissionErrors } from "./permissions.js";
 import type { Peer, SyncMessage } from "./sync.js";
-import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
+import {
+  DisconnectedError,
+  SyncManager,
+  emptyKnownState,
+  hwrServerPeerSelector,
+} from "./sync.js";
 
 import {
   getContentMessageSize,
@@ -163,6 +168,7 @@ export {
   LogLevel,
   base64URLtoBytes,
   bytesToBase64url,
+  hwrServerPeerSelector,
 };
 
 export type {

package/src/localNode.ts

@@ -97,6 +97,10 @@ export class LocalNode {
     this.storage = undefined;
   }
 
+  hasCoValue(id: RawCoID) {
+    return this.coValues.has(id);
+  }
+
   getCoValue(id: RawCoID) {
     let entry = this.coValues.get(id);
 
@@ -479,8 +483,14 @@ export class LocalNode {
       return branch.getCurrentContent() as T;
     }
 
-    // Passing skipRetry to true because otherwise creating a new branch would always take 1 retry delay
-    await this.loadCoValueCore(branch.id, undefined, true);
+    // Do a synchronous check to see if the branch exists, if not we don't need to try to load the branch
+    if (!source.hasBranch(branchName, branchOwnerID)) {
+      return source
+        .createBranch(branchName, branchOwnerID)
+        .getCurrentContent() as T;
+    }
+
+    await this.loadCoValueCore(branch.id);
 
     if (!branch.isAvailable()) {
       return source

package/src/storage/storageAsync.ts

@@ -1,7 +1,6 @@
 import {
   createContentMessage,
   exceedsRecommendedSize,
-  getTransactionSize,
 } from "../coValueContentMessage.js";
 import {
   type CoValueCore,

package/src/sync.ts

@@ -139,6 +139,13 @@ export class SyncManager {
   // the transactions have already been verified by the [trusted] peer that sent them.
   private skipVerify: boolean = false;
 
+  // When true, coValues that arrive from server peers will be ignored if they had not
+  // explicitly been requested via a load message.
+  private _ignoreUnknownCoValuesFromServers: boolean = false;
+  ignoreUnknownCoValuesFromServers() {
+    this._ignoreUnknownCoValuesFromServers = true;
+  }
+
   peersCounter = metrics.getMeter("cojson").createUpDownCounter("jazz.peers", {
     description: "Amount of connected peers",
     valueType: ValueType.INT,
@@ -191,7 +198,28 @@ export class SyncManager {
         msg,
       });
       return;
-    } else if (this.local.getCoValue(msg.id).isErroredInPeer(peer.id)) {
+    }
+
+    // Prevent core shards from storing content that belongs to other shards.
+    //
+    // This can happen because a covalue "miss" on a core shard will cause a load message to
+    // be sent to the original unsharded core. The original core, treating the peer as a client,
+    // will respond with the covalue and its dependencies. Those dependencies might not belong
+    // to this shard, so they should be ignored.
+    //
+    // TODO: remove once core has been sharded.
+    if (
+      peer.role === "server" &&
+      this._ignoreUnknownCoValuesFromServers &&
+      !this.local.hasCoValue(msg.id)
+    ) {
+      logger.warn(
+        `Ignoring message ${msg.action} on unknown coValue ${msg.id} from peer ${peer.id}`,
+      );
+      return;
+    }
+
+    if (this.local.getCoValue(msg.id).isErroredInPeer(peer.id)) {
       logger.warn(
         `Skipping message ${msg.action} on errored coValue ${msg.id} from peer ${peer.id}`,
       );
@@ -219,28 +247,7 @@ export class SyncManager {
     }
   }
 
-  sendNewContentIncludingDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, true);
-  }
-
-  sendNewContentWithoutDependencies(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-  ) {
-    this.sendNewContent(id, peer, seen, false);
-  }
-
-  private sendNewContent(
-    id: RawCoID,
-    peer: PeerState,
-    seen: Set<RawCoID> = new Set(),
-    includeDependencies: boolean,
-  ) {
+  sendNewContent(id: RawCoID, peer: PeerState, seen: Set<RawCoID> = new Set()) {
     if (seen.has(id)) {
       return;
     }
@@ -253,9 +260,10 @@ export class SyncManager {
       return;
     }
 
+    const includeDependencies = peer.role !== "server";
     if (includeDependencies) {
       for (const dependency of coValue.getDependedOnCoValues()) {
-        this.sendNewContentIncludingDependencies(dependency, peer, seen);
+        this.sendNewContent(dependency, peer, seen);
       }
     }
 
@@ -434,7 +442,7 @@ export class SyncManager {
     const coValue = this.local.getCoValue(msg.id);
 
     if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
+      this.sendNewContent(msg.id, peer);
       return;
     }
 
@@ -444,7 +452,7 @@ export class SyncManager {
 
     const handleLoadResult = () => {
       if (coValue.isAvailable()) {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
+        this.sendNewContent(msg.id, peer);
         return;
       }
 
@@ -478,11 +486,7 @@ export class SyncManager {
     }
 
     if (coValue.isAvailable()) {
-      if (peer.role === "server") {
-        this.sendNewContentWithoutDependencies(msg.id, peer);
-      } else {
-        this.sendNewContentIncludingDependencies(msg.id, peer);
-      }
+      this.sendNewContent(msg.id, peer);
     }
   }
 
@@ -777,7 +781,7 @@ export class SyncManager {
 
       // We directly forward the new content to peers that have an active subscription
       if (peer.optimisticKnownStates.has(coValue.id)) {
-        this.sendNewContentIncludingDependencies(coValue.id, peer);
+        this.sendNewContent(coValue.id, peer);
         syncedPeers.push(peer);
       } else if (
         peer.role === "server" &&
@@ -808,7 +812,7 @@ export class SyncManager {
   handleCorrection(msg: KnownStateMessage, peer: PeerState) {
     peer.setKnownState(msg.id, knownStateIn(msg));
 
-    return this.sendNewContentIncludingDependencies(msg.id, peer);
+    return this.sendNewContent(msg.id, peer);
   }
 
   private syncQueue = new LocalTransactionsSyncQueue((content) =>

package/src/tests/branching.test.ts

@@ -1,10 +1,11 @@
-import { beforeEach, describe, expect, test } from "vitest";
+import { assert, beforeEach, describe, expect, test } from "vitest";
 import {
   createTestNode,
   setupTestNode,
   loadCoValueOrFail,
 } from "./testUtils.js";
 import { expectList, expectMap, expectPlainText } from "../coValue.js";
+import { RawCoMap } from "../exports.js";
 
 let jazzCloud: ReturnType<typeof setupTestNode>;
 
@@ -94,7 +95,7 @@ describe("Branching Logic", () => {
       const result = expectMap(branch.core.mergeBranch().getCurrentContent());
 
       // Verify only one merge commit was created
-      expect(result.core.mergeCommits.length).toBe(1);
+      expect(branch.core.mergeCommits.length).toBe(1);
 
       // Verify source contains branch transactions
       expect(result.get("key1")).toBe("branchValue1");
@@ -154,7 +155,7 @@ describe("Branching Logic", () => {
       branch.core.mergeBranch();
 
       // Verify two merge commits exist
-      expect(originalMap.core.mergeCommits.length).toBe(2);
+      expect(branch.core.mergeCommits.length).toBe(2);
 
       // Verify both changes are now in original map
       expect(originalMap.get("key1")).toBe("branchValue1");
@@ -210,6 +211,8 @@ describe("Branching Logic", () => {
           .getCurrentContent(),
       );
 
+      await new Promise((resolve) => setTimeout(resolve, 5));
+
       // Add different items to second branch
       branch2.appendItems(["apples", "oranges", "carrots"]);
 
@@ -233,18 +236,20 @@ describe("Branching Logic", () => {
 
       expect(list.toJSON()).toEqual([
         "bread",
-        "cheese",
         "apples",
         "oranges",
         "carrots",
         "tomatoes",
         "cucumber",
+        "cheese",
       ]);
     });
 
-    test("should work with co.plainText when branching from different session", async () => {
-      const node = createTestNode();
-      const group = node.createGroup();
+    test("should work with co.plainText when merging the same branch twice on different sessions", async () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
       const plainText = group.createPlainText();
 
       plainText.insertAfter(0, "hello");
@@ -255,12 +260,28 @@ describe("Branching Logic", () => {
           .getCurrentContent(),
       );
 
+      branch.insertAfter("hello".length, " world");
+
+      const anotherSession = client.spawnNewSession();
+
+      const loadedBranch = await loadCoValueOrFail(
+        anotherSession.node,
+        branch.id,
+      );
+      assert(loadedBranch);
+
+      anotherSession.connectToSyncServer().peerState.gracefulShutdown();
+
       // Add more items to the branch
-      branch.insertAfter("hello".length, "world");
+      loadedBranch.insertAfter("hello world".length, " people");
 
       branch.core.mergeBranch();
+      const loadedBranchMergeResult = loadedBranch.core.mergeBranch();
 
-      expect(plainText.toString()).toEqual("helloworld");
+      anotherSession.connectToSyncServer();
+      await loadedBranchMergeResult.waitForSync();
+
+      expect(plainText.toString()).toEqual("hello world people");
     });
   });
 
@@ -527,4 +548,132 @@ describe("Branching Logic", () => {
       expect(aliceBranch.get("bob")).toBe(true);
     });
   });
+
+  test("should alias the txID when a transaction comes from a merge", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.set("key", "value");
+
+    const branch = map.core
+      .createBranch("feature-branch", group.id)
+      .getCurrentContent() as RawCoMap;
+    branch.set("branchKey", "branchValue");
+
+    const originalTxID = branch.core
+      .getValidTransactions({
+        skipBranchSource: true,
+        ignorePrivateTransactions: false,
+      })
+      .at(-1)?.txID;
+
+    branch.core.mergeBranch();
+
+    map.set("key2", "value2");
+
+    const validSortedTransactions = map.core.getValidSortedTransactions();
+
+    // Only the merged transaction should have the txId changed
+    const mergedTransactionIdx = validSortedTransactions.findIndex(
+      (tx) => tx.txID.branch,
+    );
+
+    expect(validSortedTransactions[mergedTransactionIdx - 1]?.txID.branch).toBe(
+      undefined,
+    );
+    expect(validSortedTransactions[mergedTransactionIdx]?.txID).toEqual(
+      originalTxID,
+    );
+    expect(validSortedTransactions[mergedTransactionIdx + 1]?.txID.branch).toBe(
+      undefined,
+    );
+  });
+
+  describe("hasBranch", () => {
+    test("should work when the branch owner is the source owner", () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
+      const map = group.createMap();
+
+      map.set("key", "value");
+
+      const branch = map.core.createBranch("feature-branch", group.id);
+
+      expect(map.core.hasBranch("feature-branch")).toBe(true);
+      expect(map.core.hasBranch("feature-branch", group.id)).toBe(true);
+      expect(branch.hasBranch("feature-branch")).toBe(false);
+    });
+
+    test("should work when the branch onwer is implicit", () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
+      const map = group.createMap();
+
+      map.set("key", "value");
+
+      const branch = map.core.createBranch("feature-branch");
+
+      expect(map.core.hasBranch("feature-branch")).toBe(true);
+      expect(map.core.hasBranch("feature-branch", group.id)).toBe(true);
+      expect(branch.hasBranch("feature-branch")).toBe(false);
+    });
+
+    test("should return false for non-existent branch name", () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
+      const map = group.createMap();
+
+      map.set("key", "value");
+
+      expect(map.core.hasBranch("non-existent-branch")).toBe(false);
+    });
+
+    test("should work with explicit ownerId parameter", () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
+      const map = group.createMap();
+
+      map.set("key", "value");
+
+      const differentGroup = client.node.createGroup();
+
+      map.core.createBranch("feature-branch", differentGroup.id);
+
+      // Test with explicit ownerId
+      expect(map.core.hasBranch("feature-branch", differentGroup.id)).toBe(
+        true,
+      );
+      expect(map.core.hasBranch("feature-branch")).toBe(false);
+    });
+
+    test("should work when the transactions have not been parsed yet", async () => {
+      const client = setupTestNode({
+        connected: true,
+      });
+      const group = client.node.createGroup();
+      const map = group.createMap();
+
+      map.set("key", "value");
+
+      map.core.createBranch("feature-branch", group.id);
+
+      await map.core.waitForSync();
+
+      const newSession = client.spawnNewSession();
+      const loadedMapCore = await newSession.node.loadCoValueCore(map.core.id);
+
+      expect(loadedMapCore.hasBranch("feature-branch", group.id)).toBe(true);
+    });
+  });
 });

package/src/tests/coValueCore.test.ts

@@ -8,7 +8,6 @@ import {
   vi,
 } from "vitest";
 import { CoValueCore } from "../coValueCore/coValueCore.js";
-import { Transaction } from "../coValueCore/verifiedState.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { stableStringify } from "../jsonStringify.js";
 import { LocalNode } from "../localNode.js";
@@ -24,7 +23,7 @@ import {
   tearDownTestMetricReader,
 } from "./testUtils.js";
 import { CO_VALUE_PRIORITY } from "../priority.js";
-import { determineValidTransactions } from "../permissions.js";
+import { setMaxTxSizeBytes } from "../config.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -57,6 +56,7 @@ test("transactions with wrong signature are rejected", () => {
       node.getCurrentAgent(),
       [{ hello: "world" }],
       undefined,
+      Date.now(),
     );
 
   transaction.madeAt = Date.now() + 1000;
@@ -87,6 +87,66 @@ test("transactions with wrong signature are rejected", () => {
   expect(newEntry.getValidSortedTransactions().length).toBe(0);
 });
 
+describe("transactions that exceed the byte size limit are rejected", () => {
+  beforeEach(() => {
+    setMaxTxSizeBytes(1 * 1024);
+  });
+
+  afterEach(() => {
+    setMaxTxSizeBytes(1 * 1024 * 1024);
+  });
+
+  test("makeTransaction should throw error when transaction exceeds byte size limit", () => {
+    const [agent, sessionID] = randomAgentAndSessionID();
+    const node = new LocalNode(agent.agentSecret, sessionID, Crypto);
+
+    const coValue = node.createCoValue({
+      type: "costream",
+      ruleset: { type: "unsafeAllowAll" },
+      meta: null,
+      ...Crypto.createdNowUnique(),
+    });
+
+    const largeBinaryData = "x".repeat(1024 + 100);
+
+    expect(() => {
+      coValue.makeTransaction(
+        [
+          {
+            data: largeBinaryData,
+          },
+        ],
+        "trusting",
+      );
+    }).toThrow(/Transaction is too large to be synced/);
+  });
+
+  test("makeTransaction should work for transactions under byte size limit", () => {
+    const [agent, sessionID] = randomAgentAndSessionID();
+    const node = new LocalNode(agent.agentSecret, sessionID, Crypto);
+
+    const coValue = node.createCoValue({
+      type: "costream",
+      ruleset: { type: "unsafeAllowAll" },
+      meta: null,
+      ...Crypto.createdNowUnique(),
+    });
+
+    const smallData = "Hello, world!";
+
+    const success = coValue.makeTransaction(
+      [
+        {
+          data: smallData,
+        },
+      ],
+      "trusting",
+    );
+
+    expect(success).toBe(true);
+  });
+});
+
 test("New transactions in a group correctly update owned values, including subscriptions", async () => {
   const [agent, sessionID] = randomAgentAndSessionID();
   const node = new LocalNode(agent.agentSecret, sessionID, Crypto);