cojson 0.13.18 → 0.13.20

package/package.json

@@ -25,7 +25,7 @@
   },
   "type": "module",
   "license": "MIT",
-  "version": "0.13.18",
+  "version": "0.13.20",
   "devDependencies": {
     "@opentelemetry/sdk-metrics": "^2.0.0",
     "typescript": "5.6.2"

package/src/PeerState.ts

@@ -23,15 +23,7 @@ export class PeerState {
     });
 
     this._knownStates = knownStates?.clone() ?? new PeerKnownStates();
-
-    // We assume that exchanges with storage peers are always successful
-    // hence we don't need to differentiate between knownStates and optimisticKnownStates
-    if (peer.role === "storage") {
-      this._optimisticKnownStates = "assumeInfallible";
-    } else {
-      this._optimisticKnownStates =
-        knownStates?.clone() ?? new PeerKnownStates();
-    }
+    this._optimisticKnownStates = knownStates?.clone() ?? new PeerKnownStates();
   }
 
   /**
@@ -52,13 +44,9 @@ export class PeerState {
    * The main difference with knownState is that this is updated when the content is sent to the peer without
    * waiting for any acknowledgement from the peer.
    */
-  readonly _optimisticKnownStates: PeerKnownStates | "assumeInfallible";
+  readonly _optimisticKnownStates: PeerKnownStates;
 
   get optimisticKnownStates(): ReadonlyPeerKnownStates {
-    if (this._optimisticKnownStates === "assumeInfallible") {
-      return this.knownStates;
-    }
-
     return this._optimisticKnownStates;
   }
 
@@ -76,53 +64,33 @@ export class PeerState {
 
   updateHeader(id: RawCoID, header: boolean) {
     this._knownStates.updateHeader(id, header);
-
-    if (this._optimisticKnownStates !== "assumeInfallible") {
-      this._optimisticKnownStates.updateHeader(id, header);
-    }
+    this._optimisticKnownStates.updateHeader(id, header);
   }
 
   combineWith(id: RawCoID, value: CoValueKnownState) {
     this._knownStates.combineWith(id, value);
-
-    if (this._optimisticKnownStates !== "assumeInfallible") {
-      this._optimisticKnownStates.combineWith(id, value);
-    }
+    this._optimisticKnownStates.combineWith(id, value);
   }
 
   combineOptimisticWith(id: RawCoID, value: CoValueKnownState) {
-    if (this._optimisticKnownStates === "assumeInfallible") {
-      this._knownStates.combineWith(id, value);
-    } else {
-      this._optimisticKnownStates.combineWith(id, value);
-    }
+    this._optimisticKnownStates.combineWith(id, value);
   }
 
   updateSessionCounter(id: RawCoID, sessionId: SessionID, value: number) {
     this._knownStates.updateSessionCounter(id, sessionId, value);
-
-    if (this._optimisticKnownStates !== "assumeInfallible") {
-      this._optimisticKnownStates.updateSessionCounter(id, sessionId, value);
-    }
+    this._optimisticKnownStates.updateSessionCounter(id, sessionId, value);
   }
 
   setKnownState(id: RawCoID, knownState: CoValueKnownState | "empty") {
     this._knownStates.set(id, knownState);
-
-    if (this._optimisticKnownStates !== "assumeInfallible") {
-      this._optimisticKnownStates.set(id, knownState);
-    }
+    this._optimisticKnownStates.set(id, knownState);
   }
 
   setOptimisticKnownState(
     id: RawCoID,
     knownState: CoValueKnownState | "empty",
   ) {
-    if (this._optimisticKnownStates === "assumeInfallible") {
-      this._knownStates.set(id, knownState);
-    } else {
-      this._optimisticKnownStates.set(id, knownState);
-    }
+    this._optimisticKnownStates.set(id, knownState);
   }
 
   get id() {

package/src/coValueCore/coValueCore.ts

@@ -2,11 +2,7 @@ import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { Result, err } from "neverthrow";
 import { PeerState } from "../PeerState.js";
 import { RawCoValue } from "../coValue.js";
-import {
-  ControlledAccount,
-  ControlledAccountOrAgent,
-  RawAccountID,
-} from "../coValues/account.js";
+import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
 import { RawGroup } from "../coValues/group.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
@@ -656,15 +652,15 @@ export class CoValueCore {
     a: Pick<DecryptedTransaction, "madeAt" | "txID">,
     b: Pick<DecryptedTransaction, "madeAt" | "txID">,
   ) {
-    return (
-      a.madeAt - b.madeAt ||
-      (a.txID.sessionID === b.txID.sessionID
-        ? 0
-        : a.txID.sessionID < b.txID.sessionID
-          ? -1
-          : 1) ||
-      a.txID.txIndex - b.txID.txIndex
-    );
+    if (a.madeAt !== b.madeAt) {
+      return a.madeAt - b.madeAt;
+    }
+
+    if (a.txID.sessionID === b.txID.sessionID) {
+      return a.txID.txIndex - b.txID.txIndex;
+    }
+
+    return 0;
   }
 
   getCurrentReadKey(): {
@@ -1006,6 +1002,10 @@ export class CoValueCore {
       const waitingForPeer = new Promise<void>((resolve) => {
         const markNotFound = () => {
           if (this.peers.get(peer.id)?.type === "pending") {
+            logger.warn("Timeout waiting for peer to load coValue", {
+              id: this.id,
+              peerID: peer.id,
+            });
             this.markNotFoundInPeer(peer.id);
           }
         };

package/src/localNode.ts

@@ -1,4 +1,4 @@
-import { Result, ResultAsync, err, ok, okAsync } from "neverthrow";
+import { Result, err, ok } from "neverthrow";
 import { CoID } from "./coValue.js";
 import { RawCoValue } from "./coValue.js";
 import {
@@ -223,10 +223,19 @@ export class LocalNode {
       account.set("profile", profile.id, "trusting");
     }
 
-    if (!account.get("profile")) {
+    const profileId = account.get("profile");
+
+    if (!profileId) {
       throw new Error("Must set account profile in initial migration");
     }
 
+    if (node.syncManager.hasStoragePeers()) {
+      await Promise.all([
+        node.syncManager.waitForStorageSync(account.id),
+        node.syncManager.waitForStorageSync(profileId),
+      ]);
+    }
+
     return {
       node,
       accountID: account.id,
@@ -272,11 +281,9 @@ export class LocalNode {
       if (!profileID) {
         throw new Error("Account has no profile");
       }
-      const profile = await node.load(profileID);
 
-      if (profile === "unavailable") {
-        throw new Error("Profile unavailable from all peers");
-      }
+      // Preload the profile
+      await node.load(profileID);
 
       if (migration) {
         await migration(account, node);

package/src/streamUtils.ts

@@ -6,23 +6,17 @@ export function connectedPeers(
   peer1id: PeerID,
   peer2id: PeerID,
   {
-    trace = false,
     peer1role = "client",
     peer2role = "client",
     crashOnClose = false,
   }: {
-    trace?: boolean;
     peer1role?: Peer["role"];
     peer2role?: Peer["role"];
     crashOnClose?: boolean;
   } = {},
 ): [Peer, Peer] {
-  const [from1to2Rx, from1to2Tx] = newQueuePair(
-    trace ? { traceAs: `${peer1id} -> ${peer2id}` } : undefined,
-  );
-  const [from2to1Rx, from2to1Tx] = newQueuePair(
-    trace ? { traceAs: `${peer2id} -> ${peer1id}` } : undefined,
-  );
+  const [from1to2Rx, from1to2Tx] = newQueuePair();
+  const [from2to1Rx, from2to1Tx] = newQueuePair();
 
   const peer2AsPeer: Peer = {
     id: peer2id,
@@ -43,32 +37,11 @@ export function connectedPeers(
   return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newQueuePair(
-  options: { traceAs?: string } = {},
-): [AsyncIterable<SyncMessage>, Channel<SyncMessage>] {
+export function newQueuePair(): [
+  AsyncIterable<SyncMessage>,
+  Channel<SyncMessage>,
+] {
   const channel = new Channel<SyncMessage>();
 
-  if (options.traceAs) {
-    return [
-      (async function* () {
-        for await (const msg of channel) {
-          console.debug(
-            options.traceAs,
-            JSON.stringify(
-              msg,
-              (k, v) =>
-                k === "changes" || k === "encryptedChanges"
-                  ? v.slice(0, 20) + "..."
-                  : v,
-              2,
-            ),
-          );
-          yield msg;
-        }
-      })(),
-      channel,
-    ];
-  } else {
-    return [channel.wrap(), channel];
-  }
+  return [channel.wrap(), channel];
 }

package/src/sync.ts

@@ -159,6 +159,12 @@ export class SyncManager {
     );
   }
 
+  hasStoragePeers(): boolean {
+    return this.getPeers().some(
+      (peer) => peer.role === "storage" && !peer.closed,
+    );
+  }
+
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
     if (this.local.getCoValue(msg.id).isErroredInPeer(peer.id)) {
       logger.warn(
@@ -393,10 +399,12 @@ export class SyncManager {
 
         return;
       } else {
-        // Should move the state to loading
-        this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
-          logger.error("Error loading coValue in handleLoad", { err: e });
-        });
+        // Syncronously updates the state loading is possible
+        coValue
+          .loadFromPeers(this.getServerAndStoragePeers(peer.id))
+          .catch((e) => {
+            logger.error("Error loading coValue in handleLoad", { err: e });
+          });
       }
     }
 
@@ -622,28 +630,24 @@ export class SyncManager {
 
   handleUnsubscribe(_msg: DoneMessage) {}
 
-  requestedSyncs = new Map<RawCoID, Promise<void>>();
-
-  async requestCoValueSync(coValue: CoValueCore) {
-    const promise = this.requestedSyncs.get(coValue.id);
+  requestedSyncs = new Set<RawCoID>();
+  requestCoValueSync(coValue: CoValueCore) {
+    if (this.requestedSyncs.has(coValue.id)) {
+      return;
+    }
 
-    if (promise) {
-      return promise;
-    } else {
-      const promise = new Promise<void>((resolve) => {
-        queueMicrotask(() => {
-          this.requestedSyncs.delete(coValue.id);
-          this.syncCoValue(coValue);
-          resolve();
-        });
-      });
+    queueMicrotask(() => {
+      if (this.requestedSyncs.has(coValue.id)) {
+        this.syncCoValue(coValue);
+      }
+    });
 
-      this.requestedSyncs.set(coValue.id, promise);
-      return promise;
-    }
+    this.requestedSyncs.add(coValue.id);
   }
 
   async syncCoValue(coValue: CoValueCore) {
+    this.requestedSyncs.delete(coValue.id);
+
     for (const peer of this.peersInPriorityOrder()) {
       if (peer.closed) continue;
       if (coValue.isErroredInPeer(peer.id)) continue;
@@ -674,6 +678,21 @@ export class SyncManager {
       return true;
     }
 
+    const peerState = this.peers[peerId];
+
+    // The peer has been closed, so it isn't possible to sync
+    if (!peerState || peerState.closed) {
+      return true;
+    }
+
+    // The client isn't subscribed to the coValue, so we won't sync it
+    if (
+      peerState.role === "client" &&
+      !peerState.optimisticKnownStates.has(id)
+    ) {
+      return true;
+    }
+
     return new Promise((resolve, reject) => {
       const unsubscribe = this.syncState.subscribeToPeerUpdates(
         peerId,
@@ -693,10 +712,20 @@ export class SyncManager {
     });
   }
 
+  async waitForStorageSync(id: RawCoID, timeout = 30_000) {
+    const peers = this.getPeers();
+
+    await Promise.all(
+      peers
+        .filter((peer) => peer.role === "storage")
+        .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
+    );
+  }
+
   async waitForSync(id: RawCoID, timeout = 30_000) {
     const peers = this.getPeers();
 
-    return Promise.all(
+    await Promise.all(
       peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)),
     );
   }

package/src/tests/PeerState.test.ts

@@ -174,9 +174,6 @@ describe("PeerState", () => {
   test("should dispatch to both states", () => {
     const { peerState } = setup();
     const knownStatesSpy = vi.spyOn(peerState._knownStates, "set");
-    if (peerState._optimisticKnownStates === "assumeInfallible") {
-      throw new Error("Expected normal optimisticKnownStates");
-    }
 
     const optimisticKnownStatesSpy = vi.spyOn(
       peerState._optimisticKnownStates,
@@ -195,40 +192,6 @@ describe("PeerState", () => {
     expect(optimisticKnownStatesSpy).toHaveBeenCalledWith("co_z1", state);
   });
 
-  test("should use same reference for knownStates and optimisticKnownStates for storage peers", () => {
-    const mockStoragePeer: Peer = {
-      id: "test-storage-peer",
-      role: "storage",
-      priority: 1,
-      crashOnClose: false,
-      incoming: (async function* () {})(),
-      outgoing: {
-        push: vi.fn().mockResolvedValue(undefined),
-        close: vi.fn(),
-      },
-    };
-    const peerState = new PeerState(mockStoragePeer, undefined);
-
-    // Verify they are the same reference
-    expect(peerState.knownStates).toBe(peerState.optimisticKnownStates);
-
-    // Verify that dispatching only updates one state
-    const knownStatesSpy = vi.spyOn(peerState._knownStates, "set");
-    expect(peerState._optimisticKnownStates).toBe("assumeInfallible");
-
-    const state: CoValueKnownState = {
-      id: "co_z1",
-      header: false,
-      sessions: {},
-    };
-
-    peerState.setKnownState("co_z1", state);
-
-    // Only one dispatch should happen since they're the same reference
-    expect(knownStatesSpy).toHaveBeenCalledTimes(1);
-    expect(knownStatesSpy).toHaveBeenCalledWith("co_z1", state);
-  });
-
   test("should use separate references for knownStates and optimisticKnownStates for non-storage peers", () => {
     const { peerState } = setup(); // Uses a regular peer
 

package/src/tests/SyncStateManager.test.ts

@@ -70,12 +70,12 @@ describe("SyncStateManager", () => {
     const map = group.createMap();
     map.set("key1", "value1", "trusting");
 
-    const [clientStoragePeer] = connectedPeers("clientStorage", "unusedPeer", {
-      peer1role: "client",
-      peer2role: "server",
+    const [serverPeer] = connectedPeers("serverPeer", "unusedPeer", {
+      peer1role: "server",
+      peer2role: "client",
     });
 
-    client.node.syncManager.addPeer(clientStoragePeer);
+    client.node.syncManager.addPeer(serverPeer);
 
     const subscriptionManager = client.node.syncManager.syncState;
 
@@ -86,7 +86,7 @@ describe("SyncStateManager", () => {
       updateToJazzCloudSpy,
     );
     const unsubscribe2 = subscriptionManager.subscribeToPeerUpdates(
-      clientStoragePeer.id,
+      serverPeer.id,
       updateToStorageSpy,
     );
 
@@ -115,7 +115,7 @@ describe("SyncStateManager", () => {
     );
 
     expect(updateToStorageSpy).toHaveBeenLastCalledWith(
-      emptyKnownState(map.core.id),
+      emptyKnownState(group.core.id),
       { uploaded: false },
     );
   });
@@ -247,4 +247,54 @@ describe("SyncStateManager", () => {
       ),
     ).toEqual({ uploaded: true });
   });
+
+  test("should skip closed peers", async () => {
+    const client = setupTestNode();
+    const { peerState } = client.connectToSyncServer();
+
+    peerState.gracefulShutdown();
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    await expect(map.core.waitForSync()).resolves.toBeUndefined();
+  });
+
+  test("should skip client peers that are not subscribed to the coValue", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    const client = setupTestNode();
+
+    client.connectToSyncServer({
+      syncServer: server.node,
+    });
+
+    const group = server.node.createGroup();
+    const map = group.createMap();
+
+    await map.core.waitForSync();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(false);
+  });
+
+  test("should wait for client peers that are subscribed to the coValue", async () => {
+    const server = setupTestNode({ isSyncServer: true });
+    const client = setupTestNode();
+
+    const { peerStateOnServer } = client.connectToSyncServer();
+
+    const group = server.node.createGroup();
+    const map = group.createMap();
+    map.set("key1", "value1", "trusting");
+
+    // Simulate the subscription to the coValue
+    peerStateOnServer.setKnownState(map.core.id, {
+      id: map.core.id,
+      header: true,
+      sessions: {},
+    });
+
+    await map.core.waitForSync();
+
+    expect(client.node.getCoValue(map.id).isAvailable()).toBe(true);
+  });
 });

package/src/tests/account.test.ts

@@ -1,7 +1,9 @@
 import { expect, test } from "vitest";
+import { expectAccount } from "../coValues/account.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
+import { createMockStoragePeer } from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -85,3 +87,25 @@ test("throws an error if the user tried to create an invite from an account", as
     "Cannot create invite from an account",
   );
 });
+
+test("wait for storage sync before resolving withNewlyCreatedAccount", async () => {
+  const { storage, peer } = createMockStoragePeer({
+    peerId: "account-node",
+  });
+
+  const { node, accountID } = await LocalNode.withNewlyCreatedAccount({
+    creationProps: { name: "Hermes Puggington" },
+    crypto: Crypto,
+    peersToLoadFrom: [peer],
+  });
+
+  const account = storage.getCoValue(accountID);
+
+  expect(account.isAvailable()).toBe(true);
+
+  const profile = storage.getCoValue(
+    expectAccount(account.getCurrentContent()).get("profile")!,
+  );
+
+  expect(profile.isAvailable()).toBe(true);
+});

package/src/tests/group.test.ts

@@ -750,7 +750,6 @@ describe("extend with role mapping", () => {
     const mapOnNode2 = await loadCoValueOrFail(node2.node, map.id);
 
     expect(mapOnNode2.get("test")).toEqual("Written from the admin");
-
     mapOnNode2.set("test", "Written from the inherited role");
     expect(mapOnNode2.get("test")).toEqual("Written from the inherited role");