cojson 0.8.21 → 0.8.27

package/src/coValueState.ts

@@ -1,54 +1,44 @@
+import { PeerState } from "./PeerState.js";
 import { CoValueCore } from "./coValueCore.js";
+import { RawCoID } from "./ids.js";
 import { PeerID } from "./sync.js";
 
-function createResolvablePromise<T>() {
-  let resolve!: (value: T) => void;
+export const CO_VALUE_LOADING_MAX_RETRIES = 5;
 
-  const promise = new Promise<T>((res) => {
-    resolve = res;
-  });
-
-  return { promise, resolve };
+export class CoValueUnknownState {
+  type = "unknown" as const;
 }
 
-class CoValueUnknownState {
-  type = "unknown" as const;
-  private peers: Map<
+export class CoValueLoadingState {
+  type = "loading" as const;
+  private peers = new Map<
     PeerID,
-    ReturnType<typeof createResolvablePromise<"available" | "unavailable">>
-  >;
-  private resolve: (value: "available" | "unavailable") => void;
+    ReturnType<typeof createResolvablePromise<void>>
+  >();
+  private resolveResult: (value: CoValueCore | "unavailable") => void;
 
-  ready: Promise<"available" | "unavailable">;
+  result: Promise<CoValueCore | "unavailable">;
 
   constructor(peersIds: Iterable<PeerID>) {
     this.peers = new Map();
 
     for (const peerId of peersIds) {
-      this.peers.set(
-        peerId,
-        createResolvablePromise<"available" | "unavailable">(),
-      );
+      this.peers.set(peerId, createResolvablePromise<void>());
     }
 
     const { resolve, promise } = createResolvablePromise<
-      "available" | "unavailable"
+      CoValueCore | "unavailable"
     >();
 
-    this.ready = promise;
-    this.resolve = resolve;
+    this.result = promise;
+    this.resolveResult = resolve;
   }
 
-  update(peerId: PeerID, value: "available" | "unavailable") {
+  markAsUnavailable(peerId: PeerID) {
     const entry = this.peers.get(peerId);
 
     if (entry) {
-      entry.resolve(value);
-    }
-
-    if (value === "available") {
-      this.resolve("available");
-      return;
+      entry.resolve();
     }
 
     this.peers.delete(peerId);
@@ -59,6 +49,14 @@ class CoValueUnknownState {
     }
   }
 
+  resolve(value: CoValueCore | "unavailable") {
+    this.resolveResult(value);
+    for (const entry of this.peers.values()) {
+      entry.resolve();
+    }
+    this.peers.clear();
+  }
+
   // Wait for a specific peer to have a known state
   waitForPeer(peerId: PeerID) {
     const entry = this.peers.get(peerId);
@@ -71,47 +69,267 @@ class CoValueUnknownState {
   }
 }
 
-class CoValueAvailableState {
+export class CoValueAvailableState {
   type = "available" as const;
 
   constructor(public coValue: CoValueCore) {}
 }
 
+export class CoValueUnavailableState {
+  type = "unavailable" as const;
+}
+
 type CoValueStateAction =
   | {
-      type: "not-found";
-      peerId: PeerID;
+      type: "load-requested";
+      peersIds: PeerID[];
     }
   | {
-      type: "found";
+      type: "not-found-in-peer";
       peerId: PeerID;
+    }
+  | {
+      type: "available";
       coValue: CoValueCore;
     };
 
+type CoValueStateType =
+  | CoValueUnknownState
+  | CoValueLoadingState
+  | CoValueAvailableState
+  | CoValueUnavailableState;
+
 export class CoValueState {
-  constructor(public state: CoValueUnknownState | CoValueAvailableState) {}
+  promise?: Promise<CoValueCore | "unavailable">;
+  private resolve?: (value: CoValueCore | "unavailable") => void;
+
+  constructor(
+    public id: RawCoID,
+    public state: CoValueStateType,
+  ) {}
 
-  static Unknown(peersToWaitFor: Set<PeerID>) {
-    return new CoValueState(new CoValueUnknownState(peersToWaitFor));
+  static Unknown(id: RawCoID) {
+    return new CoValueState(id, new CoValueUnknownState());
+  }
+
+  static Loading(id: RawCoID, peersIds: Iterable<PeerID>) {
+    return new CoValueState(id, new CoValueLoadingState(peersIds));
   }
 
   static Available(coValue: CoValueCore) {
-    return new CoValueState(new CoValueAvailableState(coValue));
+    return new CoValueState(coValue.id, new CoValueAvailableState(coValue));
   }
 
-  dispatch(action: CoValueStateAction) {
+  static Unavailable(id: RawCoID) {
+    return new CoValueState(id, new CoValueUnavailableState());
+  }
+
+  async getCoValue() {
     if (this.state.type === "available") {
+      return this.state.coValue;
+    }
+    if (this.state.type === "unavailable") {
+      return "unavailable";
+    }
+
+    // If we don't have a resolved state we return a new promise
+    // that will be resolved when the state will move to available or unavailable
+    if (!this.promise) {
+      const { promise, resolve } = createResolvablePromise<
+        CoValueCore | "unavailable"
+      >();
+
+      this.promise = promise;
+      this.resolve = resolve;
+    }
+
+    return this.promise;
+  }
+
+  private moveToState(value: CoValueStateType) {
+    this.state = value;
+
+    if (!this.resolve) {
+      return;
+    }
+
+    // If the state is available we resolve the promise
+    // and clear it to handle the possible transition from unavailable to available
+    if (value.type === "available") {
+      this.resolve(value.coValue);
+      this.clearPromise();
+    } else if (value.type === "unavailable") {
+      this.resolve("unavailable");
+      this.clearPromise();
+    }
+  }
+
+  private clearPromise() {
+    this.promise = undefined;
+    this.resolve = undefined;
+  }
+
+  async loadFromPeers(peers: PeerState[]) {
+    const state = this.state;
+
+    if (state.type !== "unknown" && state.type !== "unavailable") {
+      return;
+    }
+
+    if (peers.length === 0) {
       return;
     }
 
+    const doLoad = async (peersToLoadFrom: PeerState[]) => {
+      const peersWithoutErrors = getPeersWithoutErrors(
+        peersToLoadFrom,
+        this.id,
+      );
+
+      // If we are in the loading state we move to a new loading state
+      // to reset all the loading promises
+      if (this.state.type === "loading" || this.state.type === "unknown") {
+        this.moveToState(
+          new CoValueLoadingState(peersWithoutErrors.map((p) => p.id)),
+        );
+      }
+
+      // Assign the current state to a variable to not depend on the state changes
+      // that may happen while we wait for loadCoValueFromPeers to complete
+      const currentState = this.state;
+
+      // If we entered successfully the loading state, we load the coValue from the peers
+      //
+      // We may not enter the loading state if the coValue has become available in between
+      // of the retries
+      if (currentState.type === "loading") {
+        await loadCoValueFromPeers(this, peersWithoutErrors);
+
+        const result = await currentState.result;
+        return result !== "unavailable";
+      }
+
+      return currentState.type === "available";
+    };
+
+    await doLoad(peers);
+
+    // Retry loading from peers that have the retry flag enabled
+    const peersWithRetry = peers.filter((p) =>
+      p.shouldRetryUnavailableCoValues(),
+    );
+
+    if (peersWithRetry.length > 0) {
+      // We want to exit early if the coValue becomes available in between the retries
+      await Promise.race([
+        this.getCoValue(),
+        runWithRetry(
+          () => doLoad(peersWithRetry),
+          CO_VALUE_LOADING_MAX_RETRIES,
+        ),
+      ]);
+    }
+
+    // If after the retries the coValue is still loading, we consider the load failed
+    if (this.state.type === "loading") {
+      this.moveToState(new CoValueUnavailableState());
+    }
+  }
+
+  dispatch(action: CoValueStateAction) {
+    const currentState = this.state;
+
     switch (action.type) {
-      case "not-found":
-        this.state.update(action.peerId, "unavailable");
+      case "available":
+        if (currentState.type === "loading") {
+          currentState.resolve(action.coValue);
+        }
+
+        // It should be always possible to move to the available state
+        this.moveToState(new CoValueAvailableState(action.coValue));
+
         break;
-      case "found":
-        this.state.update(action.peerId, "available");
-        this.state = new CoValueAvailableState(action.coValue);
+      case "not-found-in-peer":
+        if (currentState.type === "loading") {
+          currentState.markAsUnavailable(action.peerId);
+        }
+
         break;
     }
   }
 }
+
+async function loadCoValueFromPeers(
+  coValueEntry: CoValueState,
+  peers: PeerState[],
+) {
+  for (const peer of peers) {
+    if (coValueEntry.state.type === "loading") {
+      await peer.pushOutgoingMessage({
+        action: "load",
+        id: coValueEntry.id,
+        header: false,
+        sessions: {},
+      });
+    } else if (coValueEntry.state.type === "available") {
+      await peer.pushOutgoingMessage({
+        action: "load",
+        ...coValueEntry.state.coValue.knownState(),
+      });
+    }
+
+    if (coValueEntry.state.type === "loading") {
+      await coValueEntry.state.waitForPeer(peer.id);
+    }
+  }
+}
+
+async function runWithRetry<T>(fn: () => Promise<T>, maxRetries: number) {
+  let retries = 1;
+
+  while (retries < maxRetries) {
+    /**
+     * With maxRetries of 5 we should wait:
+     * 300ms
+     * 900ms
+     * 2700ms
+     * 8100ms
+     */
+    await sleep(3 ** retries * 100);
+
+    const result = await fn();
+
+    if (result === true) {
+      return;
+    }
+
+    retries++;
+  }
+}
+
+function createResolvablePromise<T>() {
+  let resolve!: (value: T) => void;
+
+  const promise = new Promise<T>((res) => {
+    resolve = res;
+  });
+
+  return { promise, resolve };
+}
+
+function sleep(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+function getPeersWithoutErrors(peers: PeerState[], coValueId: RawCoID) {
+  return peers.filter((p) => {
+    if (p.erroredCoValues.has(coValueId)) {
+      console.error(
+        `Skipping load on errored coValue ${coValueId} from peer ${p.id}`,
+      );
+      return false;
+    }
+
+    return true;
+  });
+}

package/src/localNode.ts

@@ -1,4 +1,5 @@
 import { Result, ResultAsync, err, ok, okAsync } from "neverthrow";
+import { CoValuesStore } from "./CoValuesStore.js";
 import { CoID } from "./coValue.js";
 import { RawCoValue } from "./coValue.js";
 import {
@@ -6,7 +7,6 @@ import {
   CoValueHeader,
   CoValueUniqueness,
 } from "./coValueCore.js";
-import { CoValueState } from "./coValueState.js";
 import {
   AccountMeta,
   ControlledAccountOrAgent,
@@ -45,7 +45,7 @@ export class LocalNode {
   /** @internal */
   crypto: CryptoProvider;
   /** @internal */
-  coValues: { [key: RawCoID]: CoValueState } = {};
+  coValuesStore = new CoValuesStore();
   /** @category 3. Low-level */
   account: ControlledAccountOrAgent;
   /** @category 3. Low-level */
@@ -125,7 +125,8 @@ export class LocalNode {
     );
 
     nodeWithAccount.account = controlledAccount;
-    nodeWithAccount.coValues[controlledAccount.id] = CoValueState.Available(
+    nodeWithAccount.coValuesStore.setAsAvailable(
+      controlledAccount.id,
       controlledAccount.core,
     );
     controlledAccount.core._cachedContent = undefined;
@@ -136,7 +137,7 @@ export class LocalNode {
 
     // we shouldn't need this, but it fixes account data not syncing for new accounts
     function syncAllCoValuesAfterCreateAccount() {
-      for (const coValueEntry of Object.values(nodeWithAccount.coValues)) {
+      for (const coValueEntry of nodeWithAccount.coValuesStore.getValues()) {
         if (coValueEntry.state.type === "available") {
           void nodeWithAccount.syncManager.syncCoValue(
             coValueEntry.state.coValue,
@@ -206,7 +207,7 @@ export class LocalNode {
       node.syncManager.local = node;
 
       controlledAccount.core.node = node;
-      node.coValues[accountID] = CoValueState.Available(controlledAccount.core);
+      node.coValuesStore.setAsAvailable(accountID, controlledAccount.core);
       controlledAccount.core._cachedContent = undefined;
 
       const profileID = account.get("profile");
@@ -243,7 +244,7 @@ export class LocalNode {
     }
 
     const coValue = new CoValueCore(header, this);
-    this.coValues[coValue.id] = CoValueState.Available(coValue);
+    this.coValuesStore.setAsAvailable(coValue.id, coValue);
 
     void this.syncManager.syncCoValue(coValue);
 
@@ -253,10 +254,7 @@ export class LocalNode {
   /** @internal */
   async loadCoValueCore(
     id: RawCoID,
-    options: {
-      dontLoadFrom?: PeerID;
-      dontWaitFor?: PeerID;
-    } = {},
+    skipLoadingFromPeer?: PeerID,
   ): Promise<CoValueCore | "unavailable"> {
     if (this.crashed) {
       throw new Error("Trying to load CoValue after node has crashed", {
@@ -264,40 +262,18 @@ export class LocalNode {
       });
     }
 
-    let entry = this.coValues[id];
-    if (!entry) {
-      const peersToWaitFor = new Set(
-        Object.values(this.syncManager.peers)
-          .filter((peer) => peer.isServerOrStoragePeer())
-          .map((peer) => peer.id),
-      );
-      if (options.dontWaitFor) peersToWaitFor.delete(options.dontWaitFor);
-      entry = CoValueState.Unknown(peersToWaitFor);
-
-      this.coValues[id] = entry;
+    const entry = this.coValuesStore.get(id);
 
-      this.syncManager.loadFromPeers(id, options.dontLoadFrom).catch((e) => {
-        console.error(
-          "Error loading from peers",
-          id,
+    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
+      const peers =
+        this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
 
-          e,
-        );
+      await entry.loadFromPeers(peers).catch((e) => {
+        console.error("Error loading from peers", id, e);
       });
     }
-    if (entry.state.type === "available") {
-      return Promise.resolve(entry.state.coValue);
-    }
-
-    await entry.state.ready;
-
-    const updatedEntry = this.coValues[id];
-
-    if (updatedEntry?.state.type === "available") {
-      return Promise.resolve(updatedEntry.state.coValue);
-    }
 
-    return "unavailable";
+    return entry.getCoValue();
   }
 
   /**
@@ -318,13 +294,12 @@ export class LocalNode {
   }
 
   getLoaded<T extends RawCoValue>(id: CoID<T>): T | undefined {
-    const entry = this.coValues[id];
-    if (!entry) {
-      return undefined;
-    }
+    const entry = this.coValuesStore.get(id);
+
     if (entry.state.type === "available") {
       return entry.state.coValue.getCurrentContent() as T;
     }
+
     return undefined;
   }
 
@@ -448,15 +423,11 @@ export class LocalNode {
 
   /** @internal */
   expectCoValueLoaded(id: RawCoID, expectation?: string): CoValueCore {
-    const entry = this.coValues[id];
-    if (!entry) {
-      throw new Error(
-        `${expectation ? expectation + ": " : ""}Unknown CoValue ${id}`,
-      );
-    }
-    if (entry.state.type === "unknown") {
+    const entry = this.coValuesStore.get(id);
+
+    if (entry.state.type !== "available") {
       throw new Error(
-        `${expectation ? expectation + ": " : ""}CoValue ${id} not yet loaded`,
+        `${expectation ? expectation + ": " : ""}CoValue ${id} not yet loaded. Current state: ${entry.state.type}`,
       );
     }
     return entry.state.coValue;
@@ -650,18 +621,20 @@ export class LocalNode {
   ): LocalNode {
     const newNode = new LocalNode(account, currentSessionID, this.crypto);
 
-    const coValuesToCopy = Object.entries(this.coValues);
+    const coValuesToCopy = Array.from(this.coValuesStore.getEntries());
 
     while (coValuesToCopy.length > 0) {
       const [coValueID, entry] = coValuesToCopy[coValuesToCopy.length - 1]!;
 
-      if (entry.state.type === "unknown") {
+      if (entry.state.type !== "available") {
         coValuesToCopy.pop();
         continue;
       } else {
         const allDepsCopied = entry.state.coValue
           .getDependedOnCoValues()
-          .every((dep) => newNode.coValues[dep]?.state.type === "available");
+          .every(
+            (dep) => newNode.coValuesStore.get(dep).state.type === "available",
+          );
 
         if (!allDepsCopied) {
           // move to end of queue
@@ -675,8 +648,7 @@ export class LocalNode {
           new Map(entry.state.coValue.sessionLogs),
         );
 
-        newNode.coValues[coValueID as RawCoID] =
-          CoValueState.Available(newCoValue);
+        newNode.coValuesStore.setAsAvailable(coValueID, newCoValue);
 
         coValuesToCopy.pop();
       }