cojson 0.13.15 → 0.13.17

package/src/coValues/coList.ts

@@ -2,6 +2,7 @@ import { CoID, RawCoValue } from "../coValue.js";
 import { CoValueCore } from "../coValueCore.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
+import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
 import { RawAccountID } from "./account.js";
@@ -41,7 +42,7 @@ type DeletionEntry = {
   deletionID: OpID;
 } & DeletionOpPayload;
 
-export class RawCoListView<
+export class RawCoList<
   Item extends JsonValue = JsonValue,
   Meta extends JsonObject | null = null,
 > implements RawCoValue
@@ -81,26 +82,52 @@ export class RawCoListView<
     madeAt: number;
     opID: OpID;
   }[];
+  /** @internal */
+  totalValidTransactions = 0;
+  knownTransactions: CoValueKnownState["sessions"] = {};
+  lastValidTransaction: number | undefined;
 
   /** @internal */
   constructor(core: CoValueCore) {
     this.id = core.id as CoID<this>;
     this.core = core;
-    this.afterStart = [];
-    this.beforeEnd = [];
-    this.insertions = {};
-    this.deletionsByInsertion = {};
 
     this.insertions = {};
     this.deletionsByInsertion = {};
     this.afterStart = [];
     this.beforeEnd = [];
+    this.knownTransactions = {};
+
+    this.processNewTransactions();
+  }
+
+  processNewTransactions() {
+    const transactions = this.core.getValidSortedTransactions({
+      ignorePrivateTransactions: false,
+      knownTransactions: this.knownTransactions,
+    });
+
+    if (transactions.length === 0) {
+      return;
+    }
+
+    this.totalValidTransactions += transactions.length;
+    let lastValidTransaction: number | undefined = undefined;
+    let oldestValidTransaction: number | undefined = undefined;
+    this._cachedEntries = undefined;
+
+    for (const { txID, changes, madeAt } of transactions) {
+      lastValidTransaction = Math.max(lastValidTransaction ?? 0, madeAt);
+      oldestValidTransaction = Math.min(
+        oldestValidTransaction ?? Infinity,
+        madeAt,
+      );
+
+      this.knownTransactions[txID.sessionID] = Math.max(
+        this.knownTransactions[txID.sessionID] ?? 0,
+        txID.txIndex,
+      );
 
-    for (const {
-      txID,
-      changes,
-      madeAt,
-    } of this.core.getValidSortedTransactions()) {
       for (const [changeIdx, changeUntyped] of changes.entries()) {
         const change = changeUntyped as ListOpPayload<Item>;
 
@@ -193,6 +220,16 @@ export class RawCoListView<
         }
       }
     }
+
+    if (
+      this.lastValidTransaction &&
+      oldestValidTransaction &&
+      oldestValidTransaction < this.lastValidTransaction
+    ) {
+      this.rebuildFromCore();
+    } else {
+      this.lastValidTransaction = lastValidTransaction;
+    }
   }
 
   /** @category 6. Meta */
@@ -407,15 +444,7 @@ export class RawCoListView<
       listener(content as this);
     });
   }
-}
 
-export class RawCoList<
-    Item extends JsonValue = JsonValue,
-    Meta extends JsonObject | null = JsonObject | null,
-  >
-  extends RawCoListView<Item, Meta>
-  implements RawCoValue
-{
   /** Appends `item` after the item currently at index `after`.
    *
    * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
@@ -480,8 +509,7 @@ export class RawCoList<
     }
 
     this.core.makeTransaction(changes, privacy);
-
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /**
@@ -528,7 +556,7 @@ export class RawCoList<
       privacy,
     );
 
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /** Deletes the item at index `at`.
@@ -555,7 +583,7 @@ export class RawCoList<
       privacy,
     );
 
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   replace(
@@ -583,7 +611,7 @@ export class RawCoList<
       ],
       privacy,
     );
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /** @internal */

package/src/coValues/coMap.ts

@@ -60,6 +60,8 @@ export class RawCoMapView<
   /** @category 6. Meta */
   readonly _shape!: Shape;
 
+  totalValidTransactions = 0;
+
   /** @internal */
   constructor(
     core: CoValueCore,
@@ -133,6 +135,8 @@ export class RawCoMapView<
       }
     }
 
+    this.totalValidTransactions += newValidTransactions.length;
+
     for (const entries of changedEntries.values()) {
       entries.sort(this.core.compareTransactions);
     }

package/src/coValues/coPlainText.ts

@@ -186,7 +186,6 @@ export class RawCoPlainText<
       idx = nextIdx;
     }
     this.core.makeTransaction(ops, privacy);
-
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 }

package/src/coValues/coStream.ts

@@ -56,6 +56,7 @@ export class RawCoStreamView<
   };
   /** @internal */
   knownTransactions: CoValueKnownState["sessions"];
+  totalValidTransactions = 0;
   readonly _item!: Item;
 
   constructor(core: CoValueCore) {
@@ -96,7 +97,7 @@ export class RawCoStreamView<
   }
 
   /** @internal */
-  protected processNewTransactions() {
+  processNewTransactions() {
     const changeEntries = new Set<CoStreamItem<Item>[]>();
 
     const newValidTransactions = this.core.getValidTransactions({
@@ -109,6 +110,7 @@ export class RawCoStreamView<
     }
 
     for (const { txID, madeAt, changes } of newValidTransactions) {
+      this.totalValidTransactions++;
       for (const changeUntyped of changes) {
         const change = changeUntyped as Item;
         let entries = this.items[txID.sessionID];

package/src/localNode.ts

@@ -266,29 +266,40 @@ export class LocalNode {
       });
     }
 
-    const entry = this.coValuesStore.get(id);
+    let retries = 0;
 
-    if (
-      entry.highLevelState === "unknown" ||
-      entry.highLevelState === "unavailable"
-    ) {
-      const peers =
-        this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
+    while (true) {
+      const entry = this.coValuesStore.get(id);
 
-      if (peers.length === 0) {
-        return "unavailable";
-      }
+      if (
+        entry.highLevelState === "unknown" ||
+        entry.highLevelState === "unavailable"
+      ) {
+        const peers =
+          this.syncManager.getServerAndStoragePeers(skipLoadingFromPeer);
 
-      await entry.loadFromPeers(peers).catch((e) => {
-        logger.error("Error loading from peers", {
-          id,
-          err: e,
+        if (peers.length === 0) {
+          return "unavailable";
+        }
+
+        entry.loadFromPeers(peers).catch((e) => {
+          logger.error("Error loading from peers", {
+            id,
+            err: e,
+          });
         });
-      });
-    }
+      }
+
+      const result = await entry.getCoValue();
 
-    // TODO: What if the loading fails because in the previous loadCoValueCore call the Peer with the covalue was skipped?
-    return entry.getCoValue();
+      if (result !== "unavailable" || retries >= 1) {
+        return result;
+      }
+
+      await new Promise((resolve) => setTimeout(resolve, 300));
+
+      retries++;
+    }
   }
 
   /**

package/src/sync.ts

@@ -163,13 +163,13 @@ export class SyncManager {
         `Skipping message ${msg.action} on errored coValue ${msg.id} from peer ${peer.id}`,
       );
       return;
-    } else if (msg.id === undefined) {
-      logger.info("Received sync message with undefined id", {
+    } else if (msg.id === undefined || msg.id === null) {
+      logger.warn("Received sync message with undefined id", {
         msg,
       });
       return;
     } else if (!msg.id.startsWith("co_z")) {
-      logger.info("Received sync message with invalid id", {
+      logger.warn("Received sync message with invalid id", {
         msg,
       });
       return;

package/src/tests/coList.test.ts

@@ -1,11 +1,20 @@
-import { expect, test } from "vitest";
+import { beforeEach, expect, test } from "vitest";
 import { expectList } from "../coValue.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
-import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
+import {
+  loadCoValueOrFail,
+  randomAnonymousAccountAndSessionID,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
+beforeEach(async () => {
+  setupTestNode({ isSyncServer: true });
+});
+
 test("Empty CoList works", () => {
   const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
 
@@ -221,3 +230,176 @@ test("Items prepended to start appear with latest first", () => {
 
   expect(content.toJSON()).toEqual(["third", "second", "first"]);
 });
+
+test("mixing prepend and append", () => {
+  const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
+
+  const coValue = node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const list = expectList(coValue.getCurrentContent());
+
+  list.append(2, undefined, "trusting");
+  list.prepend(1, undefined, "trusting");
+  list.append(3, undefined, "trusting");
+
+  expect(list.toJSON()).toEqual([1, 2, 3]);
+});
+
+test("Items appended to start", () => {
+  const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
+
+  const coValue = node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const content = expectList(coValue.getCurrentContent());
+
+  content.append("first", 0, "trusting");
+  content.append("second", 0, "trusting");
+  content.append("third", 0, "trusting");
+
+  // This result is correct because "third" is appended after "first"
+  // Using the Array methods this would be the same as doing content.splice(1, 0, "third")
+  expect(content.toJSON()).toEqual(["first", "third", "second"]);
+});
+
+test("syncing appends with an older timestamp", async () => {
+  const client = setupTestNode({
+    connected: true,
+  });
+  const otherClient = setupTestNode({});
+
+  const otherClientConnection = otherClient.connectToSyncServer();
+
+  const coValue = client.node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const list = expectList(coValue.getCurrentContent());
+
+  list.append(1, undefined, "trusting");
+  list.append(2, undefined, "trusting");
+
+  const listOnOtherClient = await loadCoValueOrFail(otherClient.node, list.id);
+
+  otherClientConnection.peerState.gracefulShutdown();
+
+  listOnOtherClient.append(3, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  list.append(4, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  listOnOtherClient.append(5, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  list.append(6, undefined, "trusting");
+
+  otherClient.connectToSyncServer();
+
+  await waitFor(() => {
+    expect(list.toJSON()).toEqual([1, 2, 4, 6, 3, 5]);
+  });
+
+  expect(listOnOtherClient.toJSON()).toEqual(list.toJSON());
+});
+
+test("syncing prepends with an older timestamp", async () => {
+  const client = setupTestNode({
+    connected: true,
+  });
+  const otherClient = setupTestNode({});
+
+  const otherClientConnection = otherClient.connectToSyncServer();
+
+  const coValue = client.node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  const list = expectList(coValue.getCurrentContent());
+
+  list.prepend(1, undefined, "trusting");
+  list.prepend(2, undefined, "trusting");
+
+  const listOnOtherClient = await loadCoValueOrFail(otherClient.node, list.id);
+
+  otherClientConnection.peerState.gracefulShutdown();
+
+  listOnOtherClient.prepend(3, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  list.prepend(4, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  listOnOtherClient.prepend(5, undefined, "trusting");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  list.prepend(6, undefined, "trusting");
+
+  otherClient.connectToSyncServer();
+
+  await waitFor(() => {
+    expect(list.toJSON()).toEqual([6, 4, 5, 3, 2, 1]);
+  });
+
+  expect(listOnOtherClient.toJSON()).toEqual(list.toJSON());
+});
+
+test("totalValidTransactions should return the number of valid transactions processed", async () => {
+  const client = setupTestNode({
+    connected: true,
+  });
+  const otherClient = setupTestNode({});
+
+  const otherClientConnection = otherClient.connectToSyncServer();
+
+  const group = client.node.createGroup();
+  group.addMember("everyone", "reader");
+
+  const list = group.createList([1, 2]);
+
+  const listOnOtherClient = await loadCoValueOrFail(otherClient.node, list.id);
+
+  otherClientConnection.peerState.gracefulShutdown();
+
+  group.addMember("everyone", "writer");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  listOnOtherClient.append(3, undefined, "trusting");
+
+  expect(listOnOtherClient.toJSON()).toEqual([1, 2]);
+  expect(listOnOtherClient.totalValidTransactions).toEqual(1);
+
+  otherClient.connectToSyncServer();
+
+  await waitFor(() => {
+    expect(listOnOtherClient.core.getCurrentContent().toJSON()).toEqual([
+      1, 2, 3,
+    ]);
+  });
+
+  expect(
+    listOnOtherClient.core.getCurrentContent().totalValidTransactions,
+  ).toEqual(2);
+});

package/src/tests/coMap.test.ts

@@ -1,13 +1,23 @@
-import { expect, test } from "vitest";
+import { beforeEach, expect, test } from "vitest";
 import { expectMap } from "../coValue.js";
 import { operationToEditEntry } from "../coValues/coMap.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { LocalNode } from "../localNode.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
-import { hotSleep, randomAnonymousAccountAndSessionID } from "./testUtils.js";
+import {
+  hotSleep,
+  loadCoValueOrFail,
+  randomAnonymousAccountAndSessionID,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
+beforeEach(async () => {
+  setupTestNode({ isSyncServer: true });
+});
+
 test("Empty CoMap works", () => {
   const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
 
@@ -207,3 +217,45 @@ test("Can set items in bulk with assign", () => {
     key3: "assign3",
   });
 });
+
+test("totalValidTransactions should return the number of valid transactions processed", async () => {
+  const client = setupTestNode({
+    connected: true,
+  });
+  const otherClient = setupTestNode({});
+
+  const otherClientConnection = otherClient.connectToSyncServer();
+
+  const group = client.node.createGroup();
+  group.addMember("everyone", "reader");
+
+  const map = group.createMap({ fromClient: true });
+
+  const mapOnOtherClient = await loadCoValueOrFail(otherClient.node, map.id);
+
+  otherClientConnection.peerState.gracefulShutdown();
+
+  group.addMember("everyone", "writer");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  mapOnOtherClient.set("fromOtherClient", true, "trusting");
+
+  expect(mapOnOtherClient.totalValidTransactions).toEqual(1);
+  expect(mapOnOtherClient.toJSON()).toEqual({
+    fromClient: true,
+  });
+
+  otherClient.connectToSyncServer();
+
+  await waitFor(() => {
+    expect(mapOnOtherClient.core.getCurrentContent().toJSON()).toEqual({
+      fromClient: true,
+      fromOtherClient: true,
+    });
+  });
+
+  expect(
+    mapOnOtherClient.core.getCurrentContent().totalValidTransactions,
+  ).toEqual(2);
+});

package/src/tests/coStream.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, test } from "vitest";
+import { beforeEach, describe, expect, test } from "vitest";
 import { expectStream } from "../coValue.js";
 import { MAX_RECOMMENDED_TX_SIZE } from "../coValueCore.js";
 import {
@@ -10,10 +10,19 @@ import {
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
 import { SessionID } from "../ids.js";
 import { LocalNode } from "../localNode.js";
-import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
+import {
+  loadCoValueOrFail,
+  randomAnonymousAccountAndSessionID,
+  setupTestNode,
+  waitFor,
+} from "./testUtils.js";
 
 const Crypto = await WasmCrypto.create();
 
+beforeEach(async () => {
+  setupTestNode({ isSyncServer: true });
+});
+
 test("Empty CoStream works", () => {
   const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);
 
@@ -252,6 +261,51 @@ test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we s
   );
 });
 
+test("totalValidTransactions should return the number of valid transactions processed", async () => {
+  const client = setupTestNode({
+    connected: true,
+  });
+  const otherClient = setupTestNode({});
+
+  const otherClientConnection = otherClient.connectToSyncServer();
+
+  const group = client.node.createGroup();
+  group.addMember("everyone", "reader");
+
+  const stream = group.createStream();
+  stream.push(1, "trusting");
+
+  const streamOnOtherClient = await loadCoValueOrFail(
+    otherClient.node,
+    stream.id,
+  );
+
+  otherClientConnection.peerState.gracefulShutdown();
+
+  group.addMember("everyone", "writer");
+
+  await new Promise((resolve) => setTimeout(resolve, 50));
+
+  streamOnOtherClient.push(2, "trusting");
+
+  expect(streamOnOtherClient.totalValidTransactions).toEqual(1);
+  expect(Object.keys(streamOnOtherClient.toJSON()).length).toEqual(1);
+
+  otherClient.connectToSyncServer();
+
+  await waitFor(() => {
+    expect(
+      Object.keys(
+        streamOnOtherClient.core.getCurrentContent().toJSON() as object,
+      ).length,
+    ).toEqual(2);
+  });
+
+  expect(
+    streamOnOtherClient.core.getCurrentContent().totalValidTransactions,
+  ).toEqual(2);
+});
+
 describe("isBinaryStreamEnded", () => {
   function setup() {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID(), Crypto);