cojson 0.8.3 → 0.8.11

package/src/localNode.ts

@@ -28,10 +28,11 @@ import {
 import { RawCoValue } from "./coValue.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { err, ok, okAsync, Result, ResultAsync } from "neverthrow";
+import { CoValueState } from "./coValueState.js";
 
 /** A `LocalNode` represents a local view of a set of loaded `CoValue`s, from the perspective of a particular account (or primitive cryptographic agent).
 
-A `LocalNode` can have peers that it syncs to, for example some form of local persistence, or a sync server, such as `sync.jazz.tools` (Jazz Global Mesh).
+A `LocalNode` can have peers that it syncs to, for example some form of local persistence, or a sync server, such as `cloud.jazz.tools` (Jazz Cloud).
 
 @example
 You typically get hold of a `LocalNode` using `jazz-react`'s `useJazz()`:
@@ -130,10 +131,9 @@ export class LocalNode {
         );
 
         nodeWithAccount.account = controlledAccount;
-        nodeWithAccount.coValues[controlledAccount.id] = {
-            state: "loaded",
-            coValue: controlledAccount.core,
-        };
+        nodeWithAccount.coValues[controlledAccount.id] = CoValueState.Available(
+            controlledAccount.core,
+        );
         controlledAccount.core._cachedContent = undefined;
 
         if (!controlledAccount.get("profile")) {
@@ -145,9 +145,9 @@ export class LocalNode {
             for (const coValueEntry of Object.values(
                 nodeWithAccount.coValues,
             )) {
-                if (coValueEntry.state === "loaded") {
+                if (coValueEntry.state.type === "available") {
                     void nodeWithAccount.syncManager.syncCoValue(
-                        coValueEntry.coValue,
+                        coValueEntry.state.coValue,
                     );
                 }
             }
@@ -214,10 +214,9 @@ export class LocalNode {
             node.syncManager.local = node;
 
             controlledAccount.core.node = node;
-            node.coValues[accountID] = {
-                state: "loaded",
-                coValue: controlledAccount.core,
-            };
+            node.coValues[accountID] = CoValueState.Available(
+                controlledAccount.core,
+            );
             controlledAccount.core._cachedContent = undefined;
 
             const profileID = account.get("profile");
@@ -257,7 +256,7 @@ export class LocalNode {
         }
 
         const coValue = new CoValueCore(header, this);
-        this.coValues[coValue.id] = { state: "loaded", coValue: coValue };
+        this.coValues[coValue.id] = CoValueState.Available(coValue);
 
         void this.syncManager.syncCoValue(coValue);
 
@@ -270,7 +269,6 @@ export class LocalNode {
         options: {
             dontLoadFrom?: PeerID;
             dontWaitFor?: PeerID;
-            onProgress?: (progress: number) => void;
         } = {},
     ): Promise<CoValueCore | "unavailable"> {
         if (this.crashed) {
@@ -287,7 +285,7 @@ export class LocalNode {
                     .map((peer) => peer.id),
             );
             if (options.dontWaitFor) peersToWaitFor.delete(options.dontWaitFor);
-            entry = newLoadingState(peersToWaitFor, options.onProgress);
+            entry = CoValueState.Unknown(peersToWaitFor);
 
             this.coValues[id] = entry;
 
@@ -302,10 +300,19 @@ export class LocalNode {
                     );
                 });
         }
-        if (entry.state === "loaded") {
-            return Promise.resolve(entry.coValue);
+        if (entry.state.type === "available") {
+            return Promise.resolve(entry.state.coValue);
         }
-        return entry.done;
+
+        await entry.state.ready;
+
+        const updatedEntry = this.coValues[id];
+
+        if (updatedEntry?.state.type === "available") {
+            return Promise.resolve(updatedEntry.state.coValue);
+        }
+
+        return "unavailable";
     }
 
     /**
@@ -315,11 +322,8 @@ export class LocalNode {
      *
      * @category 3. Low-level
      */
-    async load<T extends RawCoValue>(
-        id: CoID<T>,
-        onProgress?: (progress: number) => void,
-    ): Promise<T | "unavailable"> {
-        const core = await this.loadCoValueCore(id, { onProgress });
+    async load<T extends RawCoValue>(id: CoID<T>): Promise<T | "unavailable"> {
+        const core = await this.loadCoValueCore(id);
 
         if (core === "unavailable") {
             return "unavailable";
@@ -333,8 +337,8 @@ export class LocalNode {
         if (!entry) {
             return undefined;
         }
-        if (entry.state === "loaded") {
-            return entry.coValue.getCurrentContent() as T;
+        if (entry.state.type === "available") {
+            return entry.state.coValue.getCurrentContent() as T;
         }
         return undefined;
     }
@@ -465,14 +469,14 @@ export class LocalNode {
                 `${expectation ? expectation + ": " : ""}Unknown CoValue ${id}`,
             );
         }
-        if (entry.state === "loading") {
+        if (entry.state.type === "unknown") {
             throw new Error(
                 `${
                     expectation ? expectation + ": " : ""
                 }CoValue ${id} not yet loaded`,
             );
         }
-        return entry.coValue;
+        return entry.state.coValue;
     }
 
     /** @internal */
@@ -546,7 +550,18 @@ export class LocalNode {
             return ok(id);
         }
 
-        const coValue = this.expectCoValueLoaded(id, expectation);
+        let coValue: CoValueCore;
+
+        try {
+            coValue = this.expectCoValueLoaded(id, expectation);
+        } catch (e) {
+            return err({
+                type: "ErrorLoadingCoValueCore",
+                expectation,
+                id,
+                error: e,
+            } satisfies LoadCoValueCoreError);
+        }
 
         if (
             coValue.header.type !== "comap" ||
@@ -662,13 +677,16 @@ export class LocalNode {
             const [coValueID, entry] =
                 coValuesToCopy[coValuesToCopy.length - 1]!;
 
-            if (entry.state === "loading") {
+            if (entry.state.type === "unknown") {
                 coValuesToCopy.pop();
                 continue;
             } else {
-                const allDepsCopied = entry.coValue
+                const allDepsCopied = entry.state.coValue
                     .getDependedOnCoValues()
-                    .every((dep) => newNode.coValues[dep]?.state === "loaded");
+                    .every(
+                        (dep) =>
+                            newNode.coValues[dep]?.state.type === "available",
+                    );
 
                 if (!allDepsCopied) {
                     // move to end of queue
@@ -677,15 +695,13 @@ export class LocalNode {
                 }
 
                 const newCoValue = new CoValueCore(
-                    entry.coValue.header,
+                    entry.state.coValue.header,
                     newNode,
-                    new Map(entry.coValue.sessionLogs),
+                    new Map(entry.state.coValue.sessionLogs),
                 );
 
-                newNode.coValues[coValueID as RawCoID] = {
-                    state: "loaded",
-                    coValue: newCoValue,
-                };
+                newNode.coValues[coValueID as RawCoID] =
+                    CoValueState.Available(newCoValue);
 
                 coValuesToCopy.pop();
             }
@@ -711,30 +727,6 @@ export class LocalNode {
     }
 }
 
-/** @internal */
-type CoValueState =
-    | {
-          state: "loading";
-          done: Promise<CoValueCore | "unavailable">;
-          resolve: (coValue: CoValueCore | "unavailable") => void;
-          onProgress?: (progress: number) => void;
-          firstPeerState: {
-              [peerID: string]:
-                  | {
-                        type: "waiting";
-                        done: Promise<void>;
-                        resolve: () => void;
-                    }
-                  | { type: "available" }
-                  | { type: "unavailable" };
-          };
-      }
-    | {
-          state: "loaded";
-          coValue: CoValueCore;
-          onProgress?: (progress: number) => void;
-      };
-
 export type LoadCoValueCoreError = {
     type: "ErrorLoadingCoValueCore";
     error: unknown;
@@ -759,31 +751,3 @@ export type ResolveAccountAgentError =
     | LoadCoValueCoreError
     | AccountUnavailableFromAllPeersError
     | UnexpectedlyNotAccountError;
-
-/** @internal */
-export function newLoadingState(
-    currentPeerIds: Set<PeerID>,
-    onProgress?: (progress: number) => void,
-): CoValueState {
-    let resolve: (coValue: CoValueCore | "unavailable") => void;
-
-    const promise = new Promise<CoValueCore | "unavailable">((r) => {
-        resolve = r;
-    });
-
-    return {
-        state: "loading",
-        done: promise,
-        resolve: resolve!,
-        onProgress,
-        firstPeerState: Object.fromEntries(
-            [...currentPeerIds].map((id) => {
-                let resolve: () => void;
-                const done = new Promise<void>((r) => {
-                    resolve = r;
-                });
-                return [id, { type: "waiting", done, resolve: resolve! }];
-            }),
-        ),
-    };
-}

package/src/storage/index.ts

@@ -79,7 +79,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
                     if (msg.action === "content") {
                         await this.handleNewContent(msg);
-                    } else {
+                    } else if (msg.action === 'load' || msg.action === 'known') {
                         await this.sendNewContent(msg.id, msg, undefined);
                     }
                 } catch (e) {

package/src/sync.ts

@@ -1,7 +1,8 @@
 import { Signature } from "./crypto/crypto.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
-import { LocalNode, newLoadingState } from "./localNode.js";
+import { LocalNode } from "./localNode.js";
+import { CoValueState } from "./coValueState.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { PeerState } from "./PeerState.js";
 import { CoValuePriority } from "./priority.js";
@@ -131,8 +132,9 @@ export class SyncManager {
             (peer) => peer.id !== forPeer && peer.role === "server",
         );
 
+        const coValueEntry = this.local.coValues[id];
+
         for (const peer of eligiblePeers) {
-            // console.log("loading", id, "from", peer.id);
             await peer.pushOutgoingMessage({
                 action: "load",
                 id: id,
@@ -140,44 +142,9 @@ export class SyncManager {
                 sessions: {},
             });
 
-            const coValueEntry = this.local.coValues[id];
-            if (coValueEntry?.state !== "loading") {
-                continue;
+            if (coValueEntry?.state.type === "unknown") {
+                await coValueEntry.state.waitForPeer(peer.id);
             }
-            const firstStateEntry = coValueEntry.firstPeerState[peer.id];
-            if (firstStateEntry?.type !== "waiting") {
-                throw new Error("Expected firstPeerState to be waiting " + id);
-            }
-            await new Promise<void>((resolve) => {
-                // const timeout = setTimeout(() => {
-                //     if (this.local.coValues[id]?.state === "loading") {
-                //         console.warn(
-                //             "Timeout waiting for peer to load",
-                //             id,
-                //             "from",
-                //             peer.id,
-                //             "and it hasn't loaded from other peers yet"
-                //         );
-                //     }
-                //     resolve();
-                // }, 1000);
-                firstStateEntry.done
-                    .then(() => {
-                        // clearTimeout(timeout);
-                        resolve();
-                    })
-                    .catch((e) => {
-                        // clearTimeout(timeout);
-                        console.error(
-                            "Error waiting for peer to load",
-                            id,
-                            "from",
-                            peer.id,
-                            e,
-                        );
-                        resolve();
-                    });
-            });
         }
     }
 
@@ -213,7 +180,7 @@ export class SyncManager {
             throw new Error("Expected coValue entry on subscribe");
         }
 
-        if (entry.state === "loading") {
+        if (entry.state.type === "unknown") {
             this.trySendToPeer(peer, {
                 action: "load",
                 id,
@@ -225,7 +192,7 @@ export class SyncManager {
             return;
         }
 
-        const coValue = entry.coValue;
+        const coValue = entry.state.coValue;
 
         for (const id of coValue.getDependedOnCoValues()) {
             await this.subscribeToIncludingDependencies(id, peer);
@@ -274,10 +241,7 @@ export class SyncManager {
         }
     }
 
-    async sendNewContentIncludingDependencies(
-        id: RawCoID,
-        peer: PeerState,
-    ) {
+    async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
         const coValue = this.local.expectCoValueLoaded(id);
 
         await Promise.all(
@@ -289,12 +253,12 @@ export class SyncManager {
         );
 
         const newContentPieces = coValue.newContentSince(
-            peer.optimisticKnownStates[id],
+            peer.optimisticKnownStates.get(id),
         );
 
         if (newContentPieces) {
             const optimisticKnownStateBefore =
-                peer.optimisticKnownStates[id] || emptyKnownState(id);
+                peer.optimisticKnownStates.get(id) || emptyKnownState(id);
 
             const sendPieces = async () => {
                 let lastYield = performance.now();
@@ -321,14 +285,19 @@ export class SyncManager {
 
             sendPieces().catch((e) => {
                 console.error("Error sending new content piece, retrying", e);
-                peer.optimisticKnownStates[id] = optimisticKnownStateBefore;
+                peer.optimisticKnownStates.dispatch({
+                    type: "SET",
+                    id,
+                    value: optimisticKnownStateBefore ?? emptyKnownState(id),
+                });
                 return this.sendNewContentIncludingDependencies(id, peer);
             });
 
-            peer.optimisticKnownStates[id] = combinedKnownStates(
-                optimisticKnownStateBefore,
-                coValue.knownState(),
-            );
+            peer.optimisticKnownStates.dispatch({
+                type: "COMBINE_WITH",
+                id,
+                value: coValue.knownState(),
+            });
         }
     }
 
@@ -344,11 +313,10 @@ export class SyncManager {
                     // console.log("subscribing to after peer added", id, peer.id)
                     await this.subscribeToIncludingDependencies(id, peerState);
 
-                    peerState.optimisticKnownStates[id] = {
-                        id: id,
-                        header: false,
-                        sessions: {},
-                    };
+                    peerState.optimisticKnownStates.dispatch({
+                        type: "SET_AS_EMPTY",
+                        id,
+                    });
                 }
             };
             void initialSync();
@@ -412,7 +380,11 @@ export class SyncManager {
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
-        peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
+        peer.optimisticKnownStates.dispatch({
+            type: "SET",
+            id: msg.id,
+            value: knownStateIn(msg),
+        });
         let entry = this.local.coValues[msg.id];
 
         if (!entry) {
@@ -425,7 +397,7 @@ export class SyncManager {
             );
             if (eligiblePeers.length === 0) {
                 if (msg.header || Object.keys(msg.sessions).length > 0) {
-                    this.local.coValues[msg.id] = newLoadingState(
+                    this.local.coValues[msg.id] = CoValueState.Unknown(
                         new Set([peer.id]),
                     );
                     this.trySendToPeer(peer, {
@@ -452,17 +424,21 @@ export class SyncManager {
             entry = this.local.coValues[msg.id]!;
         }
 
-        if (entry.state === "loading") {
+        if (entry.state.type === "unknown") {
             // console.debug(
             //     "Waiting for loaded",
             //     msg.id,
             //     "after message from",
             //     peer.id,
             // );
-            const loaded = await entry.done;
-            // console.log("Loaded", msg.id, loaded);
+            const loaded = await entry.state.ready;
+
             if (loaded === "unavailable") {
-                peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
+                peer.optimisticKnownStates.dispatch({
+                    type: "SET",
+                    id: msg.id,
+                    value: knownStateIn(msg),
+                });
                 peer.toldKnownState.add(msg.id);
 
                 this.trySendToPeer(peer, {
@@ -485,10 +461,11 @@ export class SyncManager {
     async handleKnownState(msg: KnownStateMessage, peer: PeerState) {
         let entry = this.local.coValues[msg.id];
 
-        peer.optimisticKnownStates[msg.id] = combinedKnownStates(
-            peer.optimisticKnownStates[msg.id] || emptyKnownState(msg.id),
-            knownStateIn(msg),
-        );
+        peer.optimisticKnownStates.dispatch({
+            type: "COMBINE_WITH",
+            id: msg.id,
+            value: knownStateIn(msg),
+        });
 
         if (!entry) {
             if (msg.asDependencyOf) {
@@ -514,31 +491,17 @@ export class SyncManager {
             }
         }
 
-        if (entry.state === "loading") {
-            const availableOnPeer = peer.optimisticKnownStates[msg.id]?.header;
-            const firstPeerStateEntry = entry.firstPeerState[peer.id];
-            if (firstPeerStateEntry?.type === "waiting") {
-                firstPeerStateEntry.resolve();
-            }
-            entry.firstPeerState[peer.id] = availableOnPeer
-                ? { type: "available" }
-                : { type: "unavailable" };
-            // console.log(
-            //     "Marking",
-            //     msg.id,
-            //     "as",
-            //     entry.firstPeerState[peer.id]?.type,
-            //     "from",
-            //     peer.id
-            // );
-            if (
-                Object.values(entry.firstPeerState).every(
-                    (s) => s.type === "unavailable",
-                )
-            ) {
-                entry.resolve("unavailable");
+        if (entry.state.type === "unknown") {
+            const availableOnPeer = peer.optimisticKnownStates.get(msg.id)?.header;
+
+            if (!availableOnPeer) {
+                entry.dispatch({
+                    type: "not-found",
+                    peerId: peer.id,
+                });
             }
-            return [];
+
+            return;
         }
 
         await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
@@ -546,7 +509,7 @@ export class SyncManager {
     }
 
     async handleNewContent(msg: NewContentMessage, peer: PeerState) {
-        let entry = this.local.coValues[msg.id];
+        const entry = this.local.coValues[msg.id];
 
         if (!entry) {
             console.error(
@@ -555,46 +518,31 @@ export class SyncManager {
             return;
         }
 
-        let resolveAfterDone: ((coValue: CoValueCore) => void) | undefined;
-
-        const peerOptimisticKnownState = peer.optimisticKnownStates[msg.id];
-
-        if (!peerOptimisticKnownState) {
-            console.error(
-                "Expected optimisticKnownState to be set for coValue we receive new content for",
-            );
-            return;
-        }
+        let coValue: CoValueCore;
 
-        if (entry.state === "loading") {
+        if (entry.state.type === "unknown") {
             if (!msg.header) {
                 console.error("Expected header to be sent in first message");
                 return;
             }
 
-            const firstPeerStateEntry = entry.firstPeerState[peer.id];
-            if (firstPeerStateEntry?.type === "waiting") {
-                firstPeerStateEntry.resolve();
-                entry.firstPeerState[peer.id] = { type: "available" };
-            }
-
-            peerOptimisticKnownState.header = true;
-
-            const coValue = new CoValueCore(msg.header, this.local);
+            peer.optimisticKnownStates.dispatch({
+                type: "UPDATE_HEADER",
+                id: msg.id,
+                header: true,
+            });
 
-            resolveAfterDone = entry.resolve;
+            coValue = new CoValueCore(msg.header, this.local);
 
-            entry = {
-                state: "loaded",
-                coValue: coValue,
-                onProgress: entry.onProgress,
-            };
-
-            this.local.coValues[msg.id] = entry;
+            entry.dispatch({
+                type: "found",
+                coValue,
+                peerId: peer.id,
+            });
+        } else {
+            coValue = entry.state.coValue;
         }
 
-        const coValue = entry.coValue;
-
         let invalidStateAssumed = false;
 
         for (const [sessionID, newContentForSession] of Object.entries(
@@ -648,15 +596,13 @@ export class SyncManager {
                 );
             }
 
-            const theirTotalnTxs = Object.values(
-                peer.optimisticKnownStates[msg.id]?.sessions || {},
-            ).reduce((sum, nTxs) => sum + nTxs, 0);
-            const ourTotalnTxs = [...coValue.sessionLogs.values()].reduce(
-                (sum, session) => sum + session.transactions.length,
-                0,
-            );
-
-            entry.onProgress?.(ourTotalnTxs / theirTotalnTxs);
+            // const theirTotalnTxs = Object.values(
+            //     peer.optimisticKnownStates[msg.id]?.sessions || {},
+            // ).reduce((sum, nTxs) => sum + nTxs, 0);
+            // const ourTotalnTxs = [...coValue.sessionLogs.values()].reduce(
+            //     (sum, session) => sum + session.transactions.length,
+            //     0,
+            // );
 
             if (result.isErr()) {
                 console.error(
@@ -673,15 +619,14 @@ export class SyncManager {
                 continue;
             }
 
-            peerOptimisticKnownState.sessions[sessionID] = Math.max(
-                peerOptimisticKnownState.sessions[sessionID] || 0,
-                newContentForSession.after +
+            peer.optimisticKnownStates.dispatch({
+                type: "UPDATE_SESSION_COUNTER",
+                id: msg.id,
+                sessionId: sessionID,
+                value:
+                    newContentForSession.after +
                     newContentForSession.newTransactions.length,
-            );
-        }
-
-        if (resolveAfterDone) {
-            resolveAfterDone(coValue);
+            });
         }
 
         await this.syncCoValue(coValue);
@@ -698,7 +643,11 @@ export class SyncManager {
     }
 
     async handleCorrection(msg: KnownStateMessage, peer: PeerState) {
-        peer.optimisticKnownStates[msg.id] = msg;
+        peer.optimisticKnownStates.dispatch({
+            type: "SET",
+            id: msg.id,
+            value: knownStateIn(msg),
+        });
 
         return this.sendNewContentIncludingDependencies(msg.id, peer);
     }
@@ -740,9 +689,7 @@ export class SyncManager {
             //     });
             //     blockingSince = performance.now();
             // }
-            const optimisticKnownState = peer.optimisticKnownStates[coValue.id];
-
-            if (optimisticKnownState) {
+            if (peer.optimisticKnownStates.has(coValue.id)) {
                 await this.tellUntoldKnownStateIncludingDependencies(
                     coValue.id,
                     peer,