cojson 0.12.0 → 0.13.0

package/src/PriorityBasedMessageQueue.ts

@@ -33,10 +33,70 @@ type Tuple<T, N extends number, A extends unknown[] = []> = A extends {
 }
   ? A
   : Tuple<T, N, [...A, T]>;
-type QueueTuple = Tuple<QueueEntry[], 8>;
+
+type QueueTuple = Tuple<LinkedList<QueueEntry>, 8>;
+
+type LinkedListNode<T> = {
+  value: T;
+  next: LinkedListNode<T> | undefined;
+};
+
+/**
+ * Using a linked list to make the shift operation O(1) instead of O(n)
+ * as our queues can grow very large when the system is under pressure.
+ */
+export class LinkedList<T> {
+  head: LinkedListNode<T> | undefined = undefined;
+  tail: LinkedListNode<T> | undefined = undefined;
+  length = 0;
+
+  push(value: T) {
+    const node = { value, next: undefined };
+
+    if (this.head === undefined) {
+      this.head = node;
+      this.tail = node;
+    } else if (this.tail) {
+      this.tail.next = node;
+      this.tail = node;
+    } else {
+      throw new Error("LinkedList is corrupted");
+    }
+
+    this.length++;
+  }
+
+  shift() {
+    if (!this.head) {
+      return undefined;
+    }
+
+    const node = this.head;
+    const value = node.value;
+    this.head = node.next;
+    node.next = undefined;
+
+    if (this.head === undefined) {
+      this.tail = undefined;
+    }
+
+    this.length--;
+
+    return value;
+  }
+}
 
 export class PriorityBasedMessageQueue {
-  private queues: QueueTuple = [[], [], [], [], [], [], [], []];
+  private queues: QueueTuple = [
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+    new LinkedList<QueueEntry>(),
+  ];
   queueSizeCounter = metrics
     .getMeter("cojson")
     .createUpDownCounter("jazz.messagequeue.size", {

package/src/coValueCore.ts

@@ -199,6 +199,7 @@ export class CoValueCore {
     givenExpectedNewHash: Hash | undefined,
     newSignature: Signature,
     skipVerify: boolean = false,
+    givenNewStreamingHash?: StreamingHash,
   ): Result<true, TryAddTransactionsError> {
     return this.node
       .resolveAccountAgent(
@@ -208,42 +209,55 @@ export class CoValueCore {
       .andThen((agent) => {
         const signerID = this.crypto.getAgentSignerID(agent);
 
-        const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
-          sessionID,
-          newTransactions,
-        );
-
-        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
-          return err({
-            type: "InvalidHash",
-            id: this.id,
-            expectedNewHash,
-            givenExpectedNewHash,
-          } satisfies InvalidHashError);
-        }
-
         if (
-          skipVerify !== true &&
-          !this.crypto.verify(newSignature, expectedNewHash, signerID)
+          skipVerify === true &&
+          givenNewStreamingHash &&
+          givenExpectedNewHash
         ) {
-          return err({
-            type: "InvalidSignature",
-            id: this.id,
+          this.doAddTransactions(
+            sessionID,
+            newTransactions,
             newSignature,
+            givenExpectedNewHash,
+            givenNewStreamingHash,
+            "immediate",
+          );
+        } else {
+          const { expectedNewHash, newStreamingHash } =
+            this.expectedNewHashAfter(sessionID, newTransactions);
+
+          if (
+            givenExpectedNewHash &&
+            givenExpectedNewHash !== expectedNewHash
+          ) {
+            return err({
+              type: "InvalidHash",
+              id: this.id,
+              expectedNewHash,
+              givenExpectedNewHash,
+            } satisfies InvalidHashError);
+          }
+
+          if (!this.crypto.verify(newSignature, expectedNewHash, signerID)) {
+            return err({
+              type: "InvalidSignature",
+              id: this.id,
+              newSignature,
+              sessionID,
+              signerID,
+            } satisfies InvalidSignatureError);
+          }
+
+          this.doAddTransactions(
             sessionID,
-            signerID,
-          } satisfies InvalidSignatureError);
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash,
+            "immediate",
+          );
         }
 
-        this.doAddTransactions(
-          sessionID,
-          newTransactions,
-          newSignature,
-          expectedNewHash,
-          newStreamingHash,
-          "immediate",
-        );
-
         return ok(true as const);
       });
   }
@@ -370,40 +384,14 @@ export class CoValueCore {
     const streamingHash =
       this.sessionLogs.get(sessionID)?.streamingHash.clone() ??
       new StreamingHash(this.crypto);
-    for (const transaction of newTransactions) {
-      streamingHash.update(transaction);
-    }
-
-    const newStreamingHash = streamingHash.clone();
 
-    return {
-      expectedNewHash: streamingHash.digest(),
-      newStreamingHash,
-    };
-  }
-
-  async expectedNewHashAfterAsync(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-  ): Promise<{ expectedNewHash: Hash; newStreamingHash: StreamingHash }> {
-    const streamingHash =
-      this.sessionLogs.get(sessionID)?.streamingHash.clone() ??
-      new StreamingHash(this.crypto);
-    let before = performance.now();
     for (const transaction of newTransactions) {
       streamingHash.update(transaction);
-      const after = performance.now();
-      if (after - before > 1) {
-        await new Promise((resolve) => setTimeout(resolve, 0));
-        before = performance.now();
-      }
     }
 
-    const newStreamingHash = streamingHash.clone();
-
     return {
       expectedNewHash: streamingHash.digest(),
-      newStreamingHash,
+      newStreamingHash: streamingHash,
     };
   }
 
@@ -452,9 +440,10 @@ export class CoValueCore {
           ) as SessionID)
         : this.node.currentSessionID;
 
-    const { expectedNewHash } = this.expectedNewHashAfter(sessionID, [
-      transaction,
-    ]);
+    const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
+      sessionID,
+      [transaction],
+    );
 
     const signature = this.crypto.sign(
       this.node.account.currentSignerSecret(),
@@ -467,6 +456,7 @@ export class CoValueCore {
       expectedNewHash,
       signature,
       true,
+      newStreamingHash,
     )._unsafeUnwrap({ withStackTrace: true });
 
     if (success) {

package/src/coValueState.ts

@@ -176,11 +176,12 @@ export class CoValueState {
   async loadFromPeers(peers: PeerState[]) {
     const state = this.state;
 
-    if (state.type !== "unknown" && state.type !== "unavailable") {
+    if (state.type === "loading" || state.type === "available") {
       return;
     }
 
     if (peers.length === 0) {
+      this.moveToState(new CoValueUnavailableState());
       return;
     }
 
@@ -192,7 +193,11 @@ export class CoValueState {
 
       // If we are in the loading state we move to a new loading state
       // to reset all the loading promises
-      if (this.state.type === "loading" || this.state.type === "unknown") {
+      if (
+        this.state.type === "loading" ||
+        this.state.type === "unknown" ||
+        this.state.type === "unavailable"
+      ) {
         this.moveToState(
           new CoValueLoadingState(peersWithoutErrors.map((p) => p.id)),
         );
@@ -308,6 +313,19 @@ async function loadCoValueFromPeers(
     }
 
     if (coValueEntry.state.type === "loading") {
+      const { promise, resolve } = createResolvablePromise<void>();
+
+      /**
+       * Use a very long timeout for storage peers, because under pressure
+       * they may take a long time to consume the messages queue
+       *
+       * TODO: Track errors on storage and do not rely on timeout
+       */
+      const timeoutDuration =
+        peer.role === "storage"
+          ? CO_VALUE_LOADING_CONFIG.TIMEOUT * 10
+          : CO_VALUE_LOADING_CONFIG.TIMEOUT;
+
       const timeout = setTimeout(() => {
         if (coValueEntry.state.type === "loading") {
           logger.warn("Failed to load coValue from peer", {
@@ -319,9 +337,10 @@ async function loadCoValueFromPeers(
             type: "not-found-in-peer",
             peerId: peer.id,
           });
+          resolve();
         }
-      }, CO_VALUE_LOADING_CONFIG.TIMEOUT);
-      await coValueEntry.state.waitForPeer(peer.id);
+      }, timeoutDuration);
+      await Promise.race([promise, coValueEntry.state.waitForPeer(peer.id)]);
       clearTimeout(timeout);
     }
   }

package/src/sync.ts

@@ -23,10 +23,6 @@ export function emptyKnownState(id: RawCoID): CoValueKnownState {
   };
 }
 
-function getErrorMessage(e: unknown) {
-  return e instanceof Error ? e.message : "Unknown error";
-}
-
 export type SyncMessage =
   | LoadMessage
   | KnownStateMessage
@@ -415,7 +411,20 @@ export class SyncManager {
           entry.loadFromPeers([peer]).catch((e) => {
             logger.error("Error loading coValue in handleLoad", { err: e });
           });
+        } else {
+          // We don't have any eligible peers to load the coValue from
+          // so we send a known state back to the sender to let it know
+          // that the coValue is unavailable
+          this.trySendToPeer(peer, {
+            action: "known",
+            id: msg.id,
+            header: false,
+            sessions: {},
+          }).catch((e) => {
+            logger.error("Error sending known state back", { err: e });
+          });
         }
+
         return;
       } else {
         this.local.loadCoValueCore(msg.id, peer.id).catch((e) => {
@@ -460,6 +469,13 @@ export class SyncManager {
             err: e,
           });
         });
+    } else if (entry.state.type === "unavailable") {
+      this.trySendToPeer(peer, {
+        action: "known",
+        id: msg.id,
+        header: false,
+        sessions: {},
+      });
     }
 
     if (entry.state.type === "available") {
@@ -604,39 +620,12 @@ export class SyncManager {
         continue;
       }
 
-      const before = performance.now();
-      // eslint-disable-next-line neverthrow/must-use-result
       const result = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
         undefined,
         newContentForSession.lastSignature,
       );
-      const after = performance.now();
-      if (after - before > 80) {
-        const totalTxLength = newTransactions
-          .map((t) =>
-            t.privacy === "private"
-              ? t.encryptedChanges.length
-              : t.changes.length,
-          )
-          .reduce((a, b) => a + b, 0);
-        logger.debug(
-          `Adding incoming transactions took ${(after - before).toFixed(
-            2,
-          )}ms for ${totalTxLength} bytes = bandwidth: ${(
-            (1000 * totalTxLength) / (after - before) / (1024 * 1024)
-          ).toFixed(2)} MB/s`,
-        );
-      }
-
-      // const theirTotalnTxs = Object.values(
-      //     peer.optimisticKnownStates[msg.id]?.sessions || {},
-      // ).reduce((sum, nTxs) => sum + nTxs, 0);
-      // const ourTotalnTxs = [...coValue.sessionLogs.values()].reduce(
-      //     (sum, session) => sum + session.transactions.length,
-      //     0,
-      // );
 
       if (result.isErr()) {
         logger.error("Failed to add transactions", {
@@ -735,16 +724,10 @@ export class SyncManager {
   }
 
   async actuallySyncCoValue(coValue: CoValueCore) {
-    // let blockingSince = performance.now();
     for (const peer of this.peersInPriorityOrder()) {
       if (peer.closed) continue;
       if (peer.erroredCoValues.has(coValue.id)) continue;
-      // if (performance.now() - blockingSince > 5) {
-      //     await new Promise<void>((resolve) => {
-      //         setTimeout(resolve, 0);
-      //     });
-      //     blockingSince = performance.now();
-      // }
+
       if (peer.optimisticKnownStates.has(coValue.id)) {
         await this.tellUntoldKnownStateIncludingDependencies(coValue.id, peer);
         await this.sendNewContentIncludingDependencies(coValue.id, peer);

package/src/tests/LinkedList.test.ts

@@ -0,0 +1,96 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import { LinkedList } from "../PriorityBasedMessageQueue";
+
+describe("LinkedList", () => {
+  let list: LinkedList<number>;
+
+  beforeEach(() => {
+    list = new LinkedList<number>();
+  });
+
+  describe("initialization", () => {
+    it("should create an empty list", () => {
+      expect(list.length).toBe(0);
+      expect(list.head).toBeUndefined();
+      expect(list.tail).toBeUndefined();
+    });
+  });
+
+  describe("push", () => {
+    it("should add an element to an empty list", () => {
+      list.push(1);
+      expect(list.length).toBe(1);
+      expect(list.head?.value).toBe(1);
+      expect(list.tail?.value).toBe(1);
+    });
+
+    it("should add multiple elements in sequence", () => {
+      list.push(1);
+      list.push(2);
+      list.push(3);
+      expect(list.length).toBe(3);
+      expect(list.head?.value).toBe(1);
+      expect(list.tail?.value).toBe(3);
+    });
+  });
+
+  describe("shift", () => {
+    it("should return undefined for empty list", () => {
+      expect(list.shift()).toBeUndefined();
+      expect(list.length).toBe(0);
+      expect(list.head).toBeUndefined();
+      expect(list.tail).toBeUndefined();
+    });
+
+    it("should remove and return the first element", () => {
+      list.push(1);
+      list.push(2);
+
+      const shifted = list.shift();
+      expect(shifted).toBe(1);
+      expect(list.length).toBe(1);
+      expect(list.head?.value).toBe(2);
+      expect(list.tail?.value).toBe(2);
+    });
+
+    it("should maintain correct order when shifting multiple times", () => {
+      list.push(1);
+      list.push(2);
+      list.push(3);
+
+      expect(list.shift()).toBe(1);
+      expect(list.shift()).toBe(2);
+      expect(list.shift()).toBe(3);
+      expect(list.length).toBe(0);
+      expect(list.head).toBeUndefined();
+      expect(list.tail).toBeUndefined();
+    });
+
+    it("should handle shift after last element is removed", () => {
+      list.push(1);
+      list.shift();
+      expect(list.shift()).toBeUndefined();
+      expect(list.length).toBe(0);
+      expect(list.head).toBeUndefined();
+      expect(list.tail).toBeUndefined();
+    });
+  });
+
+  describe("edge cases", () => {
+    it("should handle push after all elements have been shifted", () => {
+      list.push(1);
+      list.shift();
+      list.push(2);
+      expect(list.length).toBe(1);
+      expect(list.shift()).toBe(2);
+    });
+
+    it("should handle alternating push and shift operations", () => {
+      list.push(1);
+      expect(list.shift()).toBe(1);
+      list.push(2);
+      expect(list.shift()).toBe(2);
+      expect(list.length).toBe(0);
+    });
+  });
+});

package/src/tests/sync.test.ts

@@ -956,27 +956,6 @@ test.skip("When a peer's incoming/readable stream closes, we remove the peer", a
     */
 });
 
-test("If we start loading a coValue before connecting to a peer that has it, it will load it once we connect", async () => {
-  const { node: node1 } = await createConnectedTestNode();
-
-  const group = node1.createGroup();
-
-  const map = group.createMap();
-  map.set("hello", "world", "trusting");
-
-  const node2 = createTestNode();
-
-  const mapOnNode2Promise = loadCoValueOrFail(node2, map.id);
-
-  expect(node2.coValuesStore.get(map.core.id).state.type).toEqual("unknown");
-
-  connectNodeToSyncServer(node2);
-
-  const mapOnNode2 = await mapOnNode2Promise;
-
-  expect(mapOnNode2.get("hello")).toEqual("world");
-});
-
 test("should keep the peer state when the peer closes", async () => {
   const client = createTestNode();
 
@@ -1726,6 +1705,45 @@ describe("loadCoValueCore with retry", () => {
     await expect(promise1).resolves.not.toBe("unavailable");
     await expect(promise2).resolves.not.toBe("unavailable");
   });
+
+  test("should load unavailable coValues after they are synced", async () => {
+    const bob = createTestNode();
+    const alice = createTestNode();
+
+    // Create a group and map on anotherClient
+    const group = alice.createGroup();
+    const map = group.createMap();
+    map.set("key1", "value1", "trusting");
+
+    // Start loading before syncing
+    const result = await bob.loadCoValueCore(map.id);
+
+    expect(result).toBe("unavailable");
+
+    connectTwoPeers(alice, bob, "server", "server");
+
+    const result2 = await bob.loadCoValueCore(map.id);
+
+    expect(result2).not.toBe("unavailable");
+  });
+
+  test("should successfully mark a coValue as unavailable if the server does not have it", async () => {
+    const bob = createTestNode();
+    const alice = createTestNode();
+    const charlie = createTestNode();
+
+    connectTwoPeers(bob, charlie, "client", "server");
+
+    // Create a group and map on anotherClient
+    const group = alice.createGroup();
+    const map = group.createMap();
+    map.set("key1", "value1", "trusting");
+
+    // Start loading before syncing
+    const result = await bob.loadCoValueCore(map.id);
+
+    expect(result).toBe("unavailable");
+  });
 });
 
 describe("waitForSyncWithPeer", () => {
@@ -1894,6 +1912,8 @@ describe("sync protocol", () => {
     const map = group.createMap();
     map.set("hello", "world", "trusting");
 
+    await map.core.waitForSync();
+
     const mapOnJazzCloud = await loadCoValueOrFail(jazzCloud, map.id);
     expect(mapOnJazzCloud.get("hello")).toEqual("world");
 
@@ -2038,6 +2058,29 @@ describe("sync protocol", () => {
           },
         },
       },
+      {
+        from: "server",
+        msg: {
+          action: "known",
+          header: true,
+          id: map.id,
+          sessions: {
+            [client.currentSessionID]: 1,
+          },
+        },
+      },
+      {
+        from: "server",
+        msg: {
+          action: "known",
+          asDependencyOf: undefined,
+          header: true,
+          id: map.id,
+          sessions: {
+            [client.currentSessionID]: 1,
+          },
+        },
+      },
     ]);
   });
 });