cojson 0.8.21 → 0.8.27

package/src/sync.ts

@@ -2,7 +2,6 @@ import { PeerState } from "./PeerState.js";
 import { SyncStateSubscriptionManager } from "./SyncStateSubscriptionManager.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
-import { CoValueState } from "./coValueState.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { LocalNode } from "./localNode.js";
@@ -79,6 +78,7 @@ export interface Peer {
   role: "peer" | "server" | "client" | "storage";
   priority?: number;
   crashOnClose: boolean;
+  deletePeerStateOnClose?: boolean;
 }
 
 export function combinedKnownStates(
@@ -135,31 +135,10 @@ export class SyncManager {
     return Object.values(this.peers);
   }
 
-  async loadFromPeers(id: RawCoID, forPeer?: PeerID) {
-    const eligiblePeers = this.peersInPriorityOrder().filter(
-      (peer) => peer.id !== forPeer && peer.isServerOrStoragePeer(),
+  getServerAndStoragePeers(excludePeerId?: PeerID): PeerState[] {
+    return this.peersInPriorityOrder().filter(
+      (peer) => peer.isServerOrStoragePeer() && peer.id !== excludePeerId,
     );
-
-    const coValueEntry = this.local.coValues[id];
-
-    for (const peer of eligiblePeers) {
-      if (peer.erroredCoValues.has(id)) {
-        console.error(
-          `Skipping load on errored coValue ${id} from peer ${peer.id}`,
-        );
-        continue;
-      }
-      await peer.pushOutgoingMessage({
-        action: "load",
-        id: id,
-        header: false,
-        sessions: {},
-      });
-
-      if (coValueEntry?.state.type === "unknown") {
-        await coValueEntry.state.waitForPeer(peer.id);
-      }
-    }
   }
 
   async handleSyncMessage(msg: SyncMessage, peer: PeerState) {
@@ -192,19 +171,10 @@ export class SyncManager {
   }
 
   async subscribeToIncludingDependencies(id: RawCoID, peer: PeerState) {
-    const entry = this.local.coValues[id];
-
-    if (!entry) {
-      throw new Error("Expected coValue entry on subscribe");
-    }
+    const entry = this.local.coValuesStore.get(id);
 
-    if (entry.state.type === "unknown") {
-      this.trySendToPeer(peer, {
-        action: "load",
-        id,
-        header: false,
-        sessions: {},
-      }).catch((e: unknown) => {
+    if (entry.state.type !== "available") {
+      entry.loadFromPeers([peer]).catch((e: unknown) => {
         console.error("Error sending load", e);
       });
       return;
@@ -334,7 +304,7 @@ export class SyncManager {
 
     if (peerState.isServerOrStoragePeer()) {
       const initialSync = async () => {
-        for (const id of Object.keys(this.local.coValues) as RawCoID[]) {
+        for (const id of this.local.coValuesStore.getKeys()) {
           // console.log("subscribing to after peer added", id, peer.id)
           await this.subscribeToIncludingDependencies(id, peerState);
 
@@ -392,6 +362,10 @@ export class SyncManager {
         const state = this.peers[peer.id];
         state?.gracefulShutdown();
         unsubscribeFromKnownStatesUpdates();
+
+        if (peer.deletePeerStateOnClose) {
+          delete this.peers[peer.id];
+        }
       });
   }
 
@@ -405,55 +379,33 @@ export class SyncManager {
       id: msg.id,
       value: knownStateIn(msg),
     });
-    let entry = this.local.coValues[msg.id];
+    const entry = this.local.coValuesStore.get(msg.id);
 
-    if (!entry) {
-      // console.log(`Loading ${msg.id} from all peers except ${peer.id}`);
+    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
+      const eligiblePeers = this.getServerAndStoragePeers(peer.id);
 
-      // special case: we should be able to solve this much more neatly
-      // with an explicit state machine in the future
-      const eligiblePeers = this.peersInPriorityOrder().filter(
-        (other) => other.id !== peer.id && other.isServerOrStoragePeer(),
-      );
       if (eligiblePeers.length === 0) {
+        // If the load request contains a header or any session data
+        // and we don't have any eligible peers to load the coValue from
+        // we try to load it from the sender because it is the only place
+        // where we can get informations about the coValue
         if (msg.header || Object.keys(msg.sessions).length > 0) {
-          this.local.coValues[msg.id] = CoValueState.Unknown(
-            new Set([peer.id]),
-          );
-          this.trySendToPeer(peer, {
-            action: "known",
-            id: msg.id,
-            header: false,
-            sessions: {},
-          }).catch((e) => {
-            console.error("Error sending known state", e);
+          entry.loadFromPeers([peer]).catch((e) => {
+            console.error("Error loading coValue in handleLoad", e);
           });
         }
         return;
       } else {
-        this.local
-          .loadCoValueCore(msg.id, {
-            dontLoadFrom: peer.id,
-            dontWaitFor: peer.id,
-          })
-          .catch((e) => {
-            console.error("Error loading coValue in handleLoad", e);
-          });
+        this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
+          console.error("Error loading coValue in handleLoad", e);
+        });
       }
-
-      entry = this.local.coValues[msg.id]!;
     }
 
-    if (entry.state.type === "unknown") {
-      // console.debug(
-      //     "Waiting for loaded",
-      //     msg.id,
-      //     "after message from",
-      //     peer.id,
-      // );
-      const loaded = await entry.state.ready;
+    if (entry.state.type === "loading") {
+      const value = await entry.getCoValue();
 
-      if (loaded === "unavailable") {
+      if (value === "unavailable") {
         peer.dispatchToKnownStates({
           type: "SET",
           id: msg.id,
@@ -474,12 +426,14 @@ export class SyncManager {
       }
     }
 
-    await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-    await this.sendNewContentIncludingDependencies(msg.id, peer);
+    if (entry.state.type === "available") {
+      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
+      await this.sendNewContentIncludingDependencies(msg.id, peer);
+    }
   }
 
   async handleKnownState(msg: KnownStateMessage, peer: PeerState) {
-    let entry = this.local.coValues[msg.id];
+    const entry = this.local.coValuesStore.get(msg.id);
 
     peer.dispatchToKnownStates({
       type: "COMBINE_WITH",
@@ -487,18 +441,22 @@ export class SyncManager {
       value: knownStateIn(msg),
     });
 
-    if (!entry) {
+    if (entry.state.type === "unknown" || entry.state.type === "unavailable") {
       if (msg.asDependencyOf) {
-        if (this.local.coValues[msg.asDependencyOf]) {
-          this.local
-            .loadCoValueCore(msg.id, { dontLoadFrom: peer.id })
-            .catch((e) => {
-              console.error(
-                `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
-                e,
-              );
-            });
-          entry = this.local.coValues[msg.id]!; // must exist after loadCoValueCore
+        const dependencyEntry = this.local.coValuesStore.get(
+          msg.asDependencyOf,
+        );
+
+        if (
+          dependencyEntry.state.type === "available" ||
+          dependencyEntry.state.type === "loading"
+        ) {
+          this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
+            console.error(
+              `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
+              e,
+            );
+          });
         } else {
           throw new Error(
             "Expected coValue dependency entry to be created, missing subscribe?",
@@ -511,12 +469,14 @@ export class SyncManager {
       }
     }
 
-    if (entry.state.type === "unknown") {
+    // The header is a boolean value that tells us if the other peer do have information about the header.
+    // If it's false in this point it means that the coValue is unavailable on the other peer.
+    if (entry.state.type !== "available") {
       const availableOnPeer = peer.optimisticKnownStates.get(msg.id)?.header;
 
       if (!availableOnPeer) {
         entry.dispatch({
-          type: "not-found",
+          type: "not-found-in-peer",
           peerId: peer.id,
         });
       }
@@ -524,23 +484,18 @@ export class SyncManager {
       return;
     }
 
-    await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
-    await this.sendNewContentIncludingDependencies(msg.id, peer);
+    if (entry.state.type === "available") {
+      await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
+      await this.sendNewContentIncludingDependencies(msg.id, peer);
+    }
   }
 
   async handleNewContent(msg: NewContentMessage, peer: PeerState) {
-    const entry = this.local.coValues[msg.id];
-
-    if (!entry) {
-      console.error(
-        `Expected coValue entry for ${msg.id} to be created on new content, missing subscribe?`,
-      );
-      return;
-    }
+    const entry = this.local.coValuesStore.get(msg.id);
 
     let coValue: CoValueCore;
 
-    if (entry.state.type === "unknown") {
+    if (entry.state.type !== "available") {
       if (!msg.header) {
         console.error("Expected header to be sent in first message");
         return;
@@ -555,9 +510,8 @@ export class SyncManager {
       coValue = new CoValueCore(msg.header, this.local);
 
       entry.dispatch({
-        type: "found",
+        type: "available",
         coValue,
-        peerId: peer.id,
       });
     } else {
       coValue = entry.state.coValue;

package/src/tests/coValueState.test.ts

@@ -0,0 +1,427 @@
+import { describe, expect, test, vi } from "vitest";
+import { PeerState } from "../PeerState";
+import { CoValueCore } from "../coValueCore";
+import { CO_VALUE_LOADING_MAX_RETRIES, CoValueState } from "../coValueState";
+import { RawCoID } from "../ids";
+import { Peer } from "../sync";
+
+describe("CoValueState", () => {
+  const mockCoValueId = "co_test123" as RawCoID;
+
+  test("should create unknown state", () => {
+    const state = CoValueState.Unknown(mockCoValueId);
+
+    expect(state.id).toBe(mockCoValueId);
+    expect(state.state.type).toBe("unknown");
+  });
+
+  test("should create loading state", () => {
+    const peerIds = ["peer1", "peer2"];
+    const state = CoValueState.Loading(mockCoValueId, peerIds);
+
+    expect(state.id).toBe(mockCoValueId);
+    expect(state.state.type).toBe("loading");
+  });
+
+  test("should create available state", async () => {
+    const mockCoValue = createMockCoValueCore(mockCoValueId);
+    const state = CoValueState.Available(mockCoValue);
+
+    expect(state.id).toBe(mockCoValueId);
+    expect(state.state.type).toBe("available");
+    expect((state.state as any).coValue).toBe(mockCoValue);
+    await expect(state.getCoValue()).resolves.toEqual(mockCoValue);
+  });
+
+  test("should handle found action", async () => {
+    const mockCoValue = createMockCoValueCore(mockCoValueId);
+    const state = CoValueState.Loading(mockCoValueId, ["peer1", "peer2"]);
+
+    const stateValuePromise = state.getCoValue();
+
+    state.dispatch({
+      type: "available",
+      coValue: mockCoValue,
+    });
+
+    const result = await state.getCoValue();
+    expect(result).toBe(mockCoValue);
+    await expect(stateValuePromise).resolves.toBe(mockCoValue);
+  });
+
+  test("should ignore actions when not in loading state", () => {
+    const state = CoValueState.Unknown(mockCoValueId);
+
+    state.dispatch({
+      type: "not-found-in-peer",
+      peerId: "peer1",
+    });
+
+    expect(state.state.type).toBe("unknown");
+  });
+
+  test("should retry loading from peers when unsuccessful", async () => {
+    vi.useFakeTimers();
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+      },
+    );
+    const peer2 = createMockPeerState(
+      {
+        id: "peer2",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer2",
+        });
+      },
+    );
+    const mockPeers = [peer1, peer2] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    // Should attempt CO_VALUE_LOADING_MAX_RETRIES retries
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(
+      CO_VALUE_LOADING_MAX_RETRIES,
+    );
+    expect(peer2.pushOutgoingMessage).toHaveBeenCalledTimes(
+      CO_VALUE_LOADING_MAX_RETRIES,
+    );
+    expect(state.state.type).toBe("unavailable");
+    await expect(state.getCoValue()).resolves.toBe("unavailable");
+
+    vi.useRealTimers();
+  });
+
+  test("should skip errored coValues when loading from peers", async () => {
+    vi.useFakeTimers();
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        peer1.erroredCoValues.set(mockCoValueId, new Error("test") as any);
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+      },
+    );
+    const peer2 = createMockPeerState(
+      {
+        id: "peer2",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer2",
+        });
+      },
+    );
+
+    const mockPeers = [peer1, peer2] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    // Should attempt CO_VALUE_LOADING_MAX_RETRIES retries
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(1);
+    expect(peer2.pushOutgoingMessage).toHaveBeenCalledTimes(
+      CO_VALUE_LOADING_MAX_RETRIES,
+    );
+    expect(state.state.type).toBe("unavailable");
+    await expect(state.getCoValue()).resolves.toBe("unavailable");
+
+    vi.useRealTimers();
+  });
+
+  test("should retry only on server peers", async () => {
+    vi.useFakeTimers();
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "storage",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+      },
+    );
+    const peer2 = createMockPeerState(
+      {
+        id: "peer2",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer2",
+        });
+      },
+    );
+    const mockPeers = [peer1, peer2] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    // Should attempt CO_VALUE_LOADING_MAX_RETRIES retries
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(1);
+    expect(peer2.pushOutgoingMessage).toHaveBeenCalledTimes(
+      CO_VALUE_LOADING_MAX_RETRIES,
+    );
+    expect(state.state.type).toBe("unavailable");
+    await expect(state.getCoValue()).resolves.toEqual("unavailable");
+
+    vi.useRealTimers();
+  });
+
+  test("should handle the coValues that become available in between of the retries", async () => {
+    vi.useFakeTimers();
+
+    let retries = 0;
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        retries++;
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+
+        if (retries === 2) {
+          setTimeout(() => {
+            state.dispatch({
+              type: "available",
+              coValue: createMockCoValueCore(mockCoValueId),
+            });
+          }, 100);
+        }
+      },
+    );
+
+    const mockPeers = [peer1] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    // Should attempt CO_VALUE_LOADING_MAX_RETRIES retries
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES + 1; i++) {
+      await vi.runAllTimersAsync();
+    }
+
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(2);
+    expect(state.state.type).toBe("available");
+    await expect(state.getCoValue()).resolves.toEqual({ id: mockCoValueId });
+    vi.useRealTimers();
+  });
+
+  test("should have a coValue as value property when becomes available after that have been marked as unavailable", async () => {
+    vi.useFakeTimers();
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+      },
+    );
+
+    const mockPeers = [peer1] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    // Should attempt CO_VALUE_LOADING_MAX_RETRIES retries
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+
+    state.dispatch({
+      type: "available",
+      coValue: createMockCoValueCore(mockCoValueId),
+    });
+
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(5);
+    expect(state.state.type).toBe("available");
+    await expect(state.getCoValue()).resolves.toEqual({ id: mockCoValueId });
+
+    vi.useRealTimers();
+  });
+
+  test("should stop retrying when value becomes available", async () => {
+    vi.useFakeTimers();
+
+    let run = 1;
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        if (run > 2) {
+          state.dispatch({
+            type: "available",
+            coValue: createMockCoValueCore(mockCoValueId),
+          });
+        }
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer1",
+        });
+        run++;
+      },
+    );
+
+    const mockPeers = [peer1] as unknown as PeerState[];
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers(mockPeers);
+
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(3);
+    expect(state.state.type).toBe("available");
+    await expect(state.getCoValue()).resolves.toEqual({ id: mockCoValueId });
+
+    vi.useRealTimers();
+  });
+
+  test("should start sending the known state to peers when available", async () => {
+    vi.useFakeTimers();
+
+    const mockCoValue = createMockCoValueCore(mockCoValueId);
+
+    const peer1 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "storage",
+      },
+      async () => {
+        state.dispatch({
+          type: "available",
+          coValue: mockCoValue,
+        });
+      },
+    );
+    const peer2 = createMockPeerState(
+      {
+        id: "peer1",
+        role: "server",
+      },
+      async () => {
+        state.dispatch({
+          type: "not-found-in-peer",
+          peerId: "peer2",
+        });
+      },
+    );
+
+    const state = CoValueState.Unknown(mockCoValueId);
+    const loadPromise = state.loadFromPeers([peer1, peer2]);
+
+    for (let i = 0; i < CO_VALUE_LOADING_MAX_RETRIES; i++) {
+      await vi.runAllTimersAsync();
+    }
+    await loadPromise;
+
+    expect(peer1.pushOutgoingMessage).toHaveBeenCalledTimes(1);
+    expect(peer2.pushOutgoingMessage).toHaveBeenCalledTimes(1);
+    expect(peer2.pushOutgoingMessage).toHaveBeenCalledWith({
+      action: "load",
+      ...mockCoValue.knownState(),
+    });
+    expect(state.state.type).toBe("available");
+    await expect(state.getCoValue()).resolves.toEqual({ id: mockCoValueId });
+
+    vi.useRealTimers();
+  });
+});
+
+function createMockPeerState(
+  peer: Partial<Peer>,
+  pushFn = () => Promise.resolve(),
+) {
+  const peerState = new PeerState(
+    {
+      id: "peer",
+      role: "server",
+      outgoing: {
+        push: pushFn,
+      },
+      ...peer,
+    } as Peer,
+    undefined,
+  );
+
+  vi.spyOn(peerState, "pushOutgoingMessage").mockImplementation(pushFn);
+
+  return peerState;
+}
+
+function createMockCoValueCore(mockCoValueId: string) {
+  // Setting the knownState as part of the prototype to simplify
+  // the equality checks
+  const mockCoValue = Object.create({
+    knownState: vi.fn().mockReturnValue({
+      id: mockCoValueId,
+      header: true,
+      sessions: {},
+    }),
+  });
+
+  mockCoValue.id = mockCoValueId;
+  return mockCoValue as unknown as CoValueCore;
+}

package/src/tests/group.test.ts

@@ -42,7 +42,7 @@ test("Can create a CoStream in a group", () => {
   expect(stream instanceof RawCoStream).toEqual(true);
 });
 
-test("Can create a BinaryCoStream in a group", () => {
+test("Can create a FileStream in a group", () => {
   const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
 
   const group = node.createGroup();