cojson 0.13.18 → 0.13.21

package/src/localNode.ts

@@ -1,4 +1,4 @@
-import { Result, ResultAsync, err, ok, okAsync } from "neverthrow";
+import { Result, err, ok } from "neverthrow";
 import { CoID } from "./coValue.js";
 import { RawCoValue } from "./coValue.js";
 import {
@@ -223,10 +223,19 @@ export class LocalNode {
       account.set("profile", profile.id, "trusting");
     }
 
-    if (!account.get("profile")) {
+    const profileId = account.get("profile");
+
+    if (!profileId) {
       throw new Error("Must set account profile in initial migration");
     }
 
+    if (node.syncManager.hasStoragePeers()) {
+      await Promise.all([
+        node.syncManager.waitForStorageSync(account.id),
+        node.syncManager.waitForStorageSync(profileId),
+      ]);
+    }
+
     return {
       node,
       accountID: account.id,
@@ -272,11 +281,9 @@ export class LocalNode {
       if (!profileID) {
         throw new Error("Account has no profile");
       }
-      const profile = await node.load(profileID);
 
-      if (profile === "unavailable") {
-        throw new Error("Profile unavailable from all peers");
-      }
+      // Preload the profile
+      await node.load(profileID);
 
       if (migration) {
         await migration(account, node);
@@ -426,32 +433,44 @@ export class LocalNode {
     groupOrOwnedValueID: CoID<T>,
     inviteSecret: InviteSecret,
   ): Promise<void> {
-    const groupOrOwnedValue = await this.load(groupOrOwnedValueID);
+    const value = await this.load(groupOrOwnedValueID);
 
-    if (groupOrOwnedValue === "unavailable") {
+    if (value === "unavailable") {
       throw new Error(
         "Trying to accept invite: Group/owned value unavailable from all peers",
       );
     }
 
-    if (
-      groupOrOwnedValue.core.verified.header.ruleset.type === "ownedByGroup"
-    ) {
-      return this.acceptInvite(
-        groupOrOwnedValue.core.verified.header.ruleset.group as CoID<RawGroup>,
-        inviteSecret,
-      );
-    } else if (
-      groupOrOwnedValue.core.verified.header.ruleset.type !== "group"
-    ) {
-      throw new Error("Can only accept invites to groups");
+    const ruleset = value.core.verified.header.ruleset;
+
+    let group: RawGroup;
+
+    if (ruleset.type === "unsafeAllowAll") {
+      throw new Error("Can only accept invites to values owned by groups");
+    }
+
+    if (ruleset.type === "ownedByGroup") {
+      const owner = await this.load(ruleset.group as CoID<RawGroup>);
+
+      if (owner === "unavailable") {
+        throw new Error(
+          "Trying to accept invite: CoValue owner unavailable from all peers",
+        );
+      }
+
+      group = expectGroup(owner);
+    } else {
+      group = expectGroup(value);
     }
 
-    const group = expectGroup(groupOrOwnedValue);
+    if (group.core.verified.header.meta?.type === "account") {
+      throw new Error("Can't accept invites to values owned by accounts");
+    }
 
     const inviteAgentSecret = this.crypto.agentSecretFromSecretSeed(
       secretSeedFromInviteSecret(inviteSecret),
     );
+
     const inviteAgentID = this.crypto.getAgentID(inviteAgentSecret);
 
     const inviteRole = await new Promise((resolve, reject) => {
@@ -486,9 +505,10 @@ export class LocalNode {
     }
 
     const groupAsInvite = expectGroup(
-      group.core.contentInClonedNodeWithDifferentAccount(
-        new ControlledAgent(inviteAgentSecret, this.crypto),
-      ),
+      this.loadCoValueAsDifferentAgent(
+        group.id,
+        inviteAgentSecret,
+      ).getCurrentContent(),
     );
 
     groupAsInvite.addMemberInternal(
@@ -517,7 +537,7 @@ export class LocalNode {
 
     if (!coValue.isAvailable()) {
       throw new Error(
-        `${expectation ? expectation + ": " : ""}CoValue ${id} not yet loaded. Current state: ${JSON.stringify(coValue)}`,
+        `${expectation ? expectation + ": " : ""}CoValue ${id} not yet loaded.`,
       );
     }
     return coValue;
@@ -614,46 +634,56 @@ export class LocalNode {
     return group;
   }
 
-  /** @internal */
-  cloneWithDifferentAccount(
-    controlledAccountOrAgent: ControlledAccountOrAgent,
-  ): LocalNode {
+  loadCoValueAsDifferentAgent(
+    id: RawCoID,
+    secret: AgentSecret,
+    accountId?: RawAccountID | AgentID,
+  ) {
+    const agent = new ControlledAgent(secret, this.crypto);
+
     const newNode = new LocalNode(
-      controlledAccountOrAgent.agentSecret,
-      this.crypto.newRandomSessionID(controlledAccountOrAgent.id),
+      secret,
+      this.crypto.newRandomSessionID(accountId || agent.id),
       this.crypto,
     );
 
-    newNode.cloneVerifiedStateFrom(this);
+    newNode.cloneVerifiedStateFrom(this, id);
 
-    return newNode;
+    return newNode.expectCoValueLoaded(id);
   }
 
   /** @internal */
-  cloneVerifiedStateFrom(otherNode: LocalNode) {
-    const coValuesToCopy = Array.from(otherNode.coValues.entries());
+  cloneVerifiedStateFrom(otherNode: LocalNode, id: RawCoID) {
+    const coValuesIdsToCopy = [id];
 
-    while (coValuesToCopy.length > 0) {
-      const [coValueID, coValue] = coValuesToCopy[coValuesToCopy.length - 1]!;
+    // Scan all the dependencies and add them to the list
+    for (let i = 0; i < coValuesIdsToCopy.length; i++) {
+      const coValueID = coValuesIdsToCopy[i]!;
+      const coValue = otherNode.getCoValue(coValueID);
 
       if (!coValue.isAvailable()) {
-        coValuesToCopy.pop();
         continue;
-      } else {
-        const allDepsCopied = coValue
-          .getDependedOnCoValues()
-          .every((dep) => this.coValues.get(dep)?.isAvailable());
-
-        if (!allDepsCopied) {
-          // move to end of queue
-          coValuesToCopy.unshift(coValuesToCopy.pop()!);
-          continue;
-        }
+      }
 
-        this.putCoValue(coValueID, coValue.verified);
+      for (const dep of coValue.getDependedOnCoValues()) {
+        coValuesIdsToCopy.push(dep);
+      }
+    }
+
+    // Copy the coValue all the dependencies by following the dependency order
+    while (coValuesIdsToCopy.length > 0) {
+      const coValueID = coValuesIdsToCopy.pop()!;
+      const coValue = otherNode.getCoValue(coValueID);
+
+      if (!coValue.isAvailable()) {
+        continue;
+      }
 
-        coValuesToCopy.pop();
+      if (this.coValues.get(coValueID)?.isAvailable()) {
+        continue;
       }
+
+      this.putCoValue(coValueID, coValue.verified);
     }
   }
 

package/src/streamUtils.ts

@@ -6,23 +6,17 @@ export function connectedPeers(
   peer1id: PeerID,
   peer2id: PeerID,
   {
-    trace = false,
     peer1role = "client",
     peer2role = "client",
     crashOnClose = false,
   }: {
-    trace?: boolean;
     peer1role?: Peer["role"];
     peer2role?: Peer["role"];
     crashOnClose?: boolean;
   } = {},
 ): [Peer, Peer] {
-  const [from1to2Rx, from1to2Tx] = newQueuePair(
-    trace ? { traceAs: `${peer1id} -> ${peer2id}` } : undefined,
-  );
-  const [from2to1Rx, from2to1Tx] = newQueuePair(
-    trace ? { traceAs: `${peer2id} -> ${peer1id}` } : undefined,
-  );
+  const [from1to2Rx, from1to2Tx] = newQueuePair();
+  const [from2to1Rx, from2to1Tx] = newQueuePair();
 
   const peer2AsPeer: Peer = {
     id: peer2id,
@@ -43,32 +37,11 @@ export function connectedPeers(
   return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newQueuePair(
-  options: { traceAs?: string } = {},
-): [AsyncIterable<SyncMessage>, Channel<SyncMessage>] {
+export function newQueuePair(): [
+  AsyncIterable<SyncMessage>,
+  Channel<SyncMessage>,
+] {
   const channel = new Channel<SyncMessage>();
 
-  if (options.traceAs) {
-    return [
-      (async function* () {
-        for await (const msg of channel) {
-          console.debug(
-            options.traceAs,
-            JSON.stringify(
-              msg,
-              (k, v) =>
-                k === "changes" || k === "encryptedChanges"
-                  ? v.slice(0, 20) + "..."
-                  : v,
-              2,
-            ),
-          );
-          yield msg;
-        }
-      })(),
-      channel,
-    ];
-  } else {
-    return [channel.wrap(), channel];
-  }
+  return [channel.wrap(), channel];
 }

package/src/sync.ts

@@ -159,6 +159,12 @@ export class SyncManager {
     );
   }
 
+  hasStoragePeers(): boolean {
+    return this.getPeers().some(
+      (peer) => peer.role === "storage" && !peer.closed,
+    );
+  }
+
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
     if (this.local.getCoValue(msg.id).isErroredInPeer(peer.id)) {
       logger.warn(
@@ -393,10 +399,12 @@ export class SyncManager {
 
         return;
       } else {
-        // Should move the state to loading
-        this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
-          logger.error("Error loading coValue in handleLoad", { err: e });
-        });
+        // Syncronously updates the state loading is possible
+        coValue
+          .loadFromPeers(this.getServerAndStoragePeers(peer.id))
+          .catch((e) => {
+            logger.error("Error loading coValue in handleLoad", { err: e });
+          });
       }
     }
 
@@ -622,28 +630,24 @@ export class SyncManager {
 
   handleUnsubscribe(_msg: DoneMessage) {}
 
-  requestedSyncs = new Map<RawCoID, Promise<void>>();
-
-  async requestCoValueSync(coValue: CoValueCore) {
-    const promise = this.requestedSyncs.get(coValue.id);
+  requestedSyncs = new Set<RawCoID>();
+  requestCoValueSync(coValue: CoValueCore) {
+    if (this.requestedSyncs.has(coValue.id)) {
+      return;
+    }
 
-    if (promise) {
-      return promise;
-    } else {
-      const promise = new Promise<void>((resolve) => {
-        queueMicrotask(() => {
-          this.requestedSyncs.delete(coValue.id);
-          this.syncCoValue(coValue);
-          resolve();
-        });
-      });
+    queueMicrotask(() => {
+      if (this.requestedSyncs.has(coValue.id)) {
+        this.syncCoValue(coValue);
+      }
+    });
 
-      this.requestedSyncs.set(coValue.id, promise);
-      return promise;
-    }
+    this.requestedSyncs.add(coValue.id);
   }
 
   async syncCoValue(coValue: CoValueCore) {
+    this.requestedSyncs.delete(coValue.id);
+
     for (const peer of this.peersInPriorityOrder()) {
       if (peer.closed) continue;
       if (coValue.isErroredInPeer(peer.id)) continue;
@@ -674,6 +678,21 @@ export class SyncManager {
       return true;
     }
 
+    const peerState = this.peers[peerId];
+
+    // The peer has been closed, so it isn't possible to sync
+    if (!peerState || peerState.closed) {
+      return true;
+    }
+
+    // The client isn't subscribed to the coValue, so we won't sync it
+    if (
+      peerState.role === "client" &&
+      !peerState.optimisticKnownStates.has(id)
+    ) {
+      return true;
+    }
+
     return new Promise((resolve, reject) => {
       const unsubscribe = this.syncState.subscribeToPeerUpdates(
         peerId,
@@ -693,10 +712,20 @@ export class SyncManager {
     });
   }
 
+  async waitForStorageSync(id: RawCoID, timeout = 30_000) {
+    const peers = this.getPeers();
+
+    await Promise.all(
+      peers
+        .filter((peer) => peer.role === "storage")
+        .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
+    );
+  }
+
   async waitForSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    return Promise.all(
+    await Promise.all(
       peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }

package/src/tests/PeerState.test.ts

@@ -174,9 +174,6 @@ describe("PeerState", () => {
   test("should dispatch to both states", () => {
     const { peerState } = setup();
     const knownStatesSpy = vi.spyOn(peerState._knownStates, "set");
-    if (peerState._optimisticKnownStates === "assumeInfallible") {
-      throw new Error("Expected normal optimisticKnownStates");
-    }
 
     const optimisticKnownStatesSpy = vi.spyOn(
       peerState._optimisticKnownStates,
@@ -195,40 +192,6 @@ describe("PeerState", () => {
     expect(optimisticKnownStatesSpy).toHaveBeenCalledWith("co_z1", state);
   });
 
-  test("should use same reference for knownStates and optimisticKnownStates for storage peers", () => {
-    const mockStoragePeer: Peer = {
-      id: "test-storage-peer",
-      role: "storage",
-      priority: 1,
-      crashOnClose: false,
-      incoming: (async function* () {})(),
-      outgoing: {
-        push: vi.fn().mockResolvedValue(undefined),
-        close: vi.fn(),
-      },
-    };
-    const peerState = new PeerState(mockStoragePeer, undefined);
-
-    // Verify they are the same reference
-    expect(peerState.knownStates).toBe(peerState.optimisticKnownStates);
-
-    // Verify that dispatching only updates one state
-    const knownStatesSpy = vi.spyOn(peerState._knownStates, "set");
-    expect(peerState._optimisticKnownStates).toBe("assumeInfallible");
-
-    const state: CoValueKnownState = {
-      id: "co_z1",
-      header: false,
-      sessions: {},
-    };
-
-    peerState.setKnownState("co_z1", state);
-
-    // Only one dispatch should happen since they're the same reference
-    expect(knownStatesSpy).toHaveBeenCalledTimes(1);
-    expect(knownStatesSpy).toHaveBeenCalledWith("co_z1", state);
-  });
-
   test("should use separate references for knownStates and optimisticKnownStates for non-storage peers", () => {
     const { peerState } = setup(); // Uses a regular peer
 

package/src/tests/SyncStateManager.test.ts

@@ -70,12 +70,12 @@ describe("SyncStateManager", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    const [clientStoragePeer] = connectedPeers("clientStorage", "unusedPeer", {
-      peer1role: "client",
-      peer2role: "server",
+    const [serverPeer] = connectedPeers("serverPeer", "unusedPeer", {
+      peer1role: "server",
+      peer2role: "client",
     });
 
-    client.node.syncManager.addPeer(clientStoragePeer);
+    client.node.syncManager.addPeer(serverPeer);
 
     const subscriptionManager = client.node.syncManager.syncState;
 
@@ -86,7 +86,7 @@ describe("SyncStateManager", () => {
       updateToJazzCloudSpy,
     );
     const unsubscribe2 = subscriptionManager.subscribeToPeerUpdates(
-      clientStoragePeer.id,
+      serverPeer.id,
       updateToStorageSpy,
     );
 
@@ -115,7 +115,7 @@ describe("SyncStateManager", () => {
     );
 
     expect(updateToStorageSpy).toHaveBeenLastCalledWith(
-      emptyKnownState(map.core.id),
+      emptyKnownState(group.core.id),
       { uploaded: false },
     );
   });
@@ -247,4 +247,54 @@ describe("SyncStateManager", () => {
       ),
     ).toEqual({ uploaded: true });
   });
+
+  test("should skip closed peers", async () => {
+    const client = setupTestNode();
+    const { peerState } = client.connectToSyncServer();
+
+    peerState.gracefulShutdown();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    await expect(map.core.waitForSync()).resolves.toBeUndefined();
+  });
+
+  test("should skip client peers that are not subscribed to the coValue", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    const client = setupTestNode();
+
+    client.connectToSyncServer({
+      syncServer: server.node,
+    });
+
+    const group = server.node.createGroup();
+    const map = group.createMap();
+
+    await map.core.waitForSync();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+  });
+
+  test("should wait for client peers that are subscribed to the coValue", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    const client = setupTestNode();
+
+    const { peerStateOnServer } = client.connectToSyncServer();
+
+    const group = server.node.createGroup();
+    const map = group.createMap();
+    map.set("key1", "value1", "trusting");
+
+    // Simulate the subscription to the coValue
+    peerStateOnServer.setKnownState(map.core.id, {
+      id: map.core.id,
+      header: true,
+      sessions: {},
+    });
+
+    await map.core.waitForSync();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+  });
 });

package/src/tests/account.test.ts

@@ -1,7 +1,9 @@
 import { expect, test } from "vitest";
+import { expectAccount } from "../coValues/account.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
+import { createMockStoragePeer } from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -85,3 +87,25 @@ test("throws an error if the user tried to create an invite from an account", as
     "Cannot create invite from an account",
   );
 });
+
+test("wait for storage sync before resolving withNewlyCreatedAccount", async () => {
+  const { storage, peer } = createMockStoragePeer({
+    peerId: "account-node",
+  });
+
+  const { node, accountID } = await LocalNode.withNewlyCreatedAccount({
+    creationProps: { name: "Hermes Puggington" },
+    crypto: Crypto,
+    peersToLoadFrom: [peer],
+  });
+
+  const account = storage.getCoValue(accountID);
+
+  expect(account.isAvailable()).toBe(true);
+
+  const profile = storage.getCoValue(
+    expectAccount(account.getCurrentContent()).get("profile")!,
+  );
+
+  expect(profile.isAvailable()).toBe(true);
+});

package/src/tests/group.test.ts

@@ -750,7 +750,6 @@ describe("extend with role mapping", () => {
     const mapOnNode2 = await loadCoValueOrFail(node2.node, map.id);
 
     expect(mapOnNode2.get("test")).toEqual("Written from the admin");
-
     mapOnNode2.set("test", "Written from the inherited role");
     expect(mapOnNode2.get("test")).toEqual("Written from the inherited role");
 

package/src/tests/permissions.test.ts

@@ -512,7 +512,6 @@ test("Admins can set group read key and then use it to create private transactio
   const { node, groupCore, admin } = newGroup();
 
   const reader1 = createAccountInNode(node);
-
   const reader2 = createAccountInNode(node);
 
   const { secret: readKey, id: readKeyID } = Crypto.newRandomKeySecret();
@@ -583,6 +582,9 @@ test("Admins can set group read key and then use it to create private transactio
     childObject.contentInClonedNodeWithDifferentAccount(reader2),
   );
 
+  // Need to copy the account coValue to the new node to be able to read the readKey
+  childObjectAsReader2.core.node.cloneVerifiedStateFrom(node, reader2.id);
+
   expect(childObjectAsReader2.core.getCurrentReadKey().secret).toEqual(readKey);
   expect(childObjectAsReader2.get("foo")).toEqual("bar");
 });
@@ -1113,9 +1115,10 @@ test("Admins can create an adminInvite, which can add an admin (high-level)", as
   const invitedAdminSecret = Crypto.newRandomAgentSecret();
   const invitedAdminID = Crypto.getAgentID(invitedAdminSecret);
 
-  const nodeAsInvitedAdmin = node.cloneWithDifferentAccount(
-    new ControlledAgent(invitedAdminSecret, Crypto),
-  );
+  const nodeAsInvitedAdmin = node.loadCoValueAsDifferentAgent(
+    group.id,
+    invitedAdminSecret,
+  ).node;
 
   await nodeAsInvitedAdmin.acceptInvite(group.id, inviteSecret);
 
@@ -1219,9 +1222,10 @@ test("Admins can create a writerInvite, which can add a writer (high-level)", as
   const invitedWriterSecret = Crypto.newRandomAgentSecret();
   const invitedWriterID = Crypto.getAgentID(invitedWriterSecret);
 
-  const nodeAsInvitedWriter = node.cloneWithDifferentAccount(
-    new ControlledAgent(invitedWriterSecret, Crypto),
-  );
+  const nodeAsInvitedWriter = node.loadCoValueAsDifferentAgent(
+    group.id,
+    invitedWriterSecret,
+  ).node;
 
   await nodeAsInvitedWriter.acceptInvite(group.id, inviteSecret);
 
@@ -1308,9 +1312,10 @@ test("Admins can create a readerInvite, which can add a reader (high-level)", as
   const invitedReaderSecret = Crypto.newRandomAgentSecret();
   const invitedReaderID = Crypto.getAgentID(invitedReaderSecret);
 
-  const nodeAsInvitedReader = node.cloneWithDifferentAccount(
-    new ControlledAgent(invitedReaderSecret, Crypto),
-  );
+  const nodeAsInvitedReader = node.loadCoValueAsDifferentAgent(
+    group.id,
+    invitedReaderSecret,
+  ).node;
 
   await nodeAsInvitedReader.acceptInvite(group.id, inviteSecret);