cojson 0.17.11 → 0.17.13

package/src/sync.ts

@@ -128,6 +128,11 @@ export function combinedKnownStates(
   };
 }
 
+export type ServerPeerSelector = (
+  id: RawCoID,
+  serverPeers: PeerState[],
+) => PeerState[];
+
 export class SyncManager {
   peers: { [key: PeerID]: PeerState } = {};
   local: LocalNode;
@@ -144,6 +149,8 @@ export class SyncManager {
   });
   private transactionsSizeHistogram: Histogram;
 
+  serverPeerSelector?: ServerPeerSelector;
+
   constructor(local: LocalNode) {
     this.local = local;
     this.syncState = new SyncStateManager(this);
@@ -163,23 +170,21 @@ export class SyncManager {
     this.skipVerify = true;
   }
 
-  peersInPriorityOrder(): PeerState[] {
-    return Object.values(this.peers).sort((a, b) => {
-      const aPriority = a.priority || 0;
-      const bPriority = b.priority || 0;
-
-      return bPriority - aPriority;
-    });
+  getPeers(id: RawCoID): PeerState[] {
+    return this.getServerPeers(id).concat(this.getClientPeers());
   }
 
-  getPeers(): PeerState[] {
-    return Object.values(this.peers);
+  getClientPeers(): PeerState[] {
+    return Object.values(this.peers).filter((peer) => peer.role === "client");
   }
 
-  getServerPeers(excludePeerId?: PeerID): PeerState[] {
-    return this.getPeers().filter(
+  getServerPeers(id: RawCoID, excludePeerId?: PeerID): PeerState[] {
+    const serverPeers = Object.values(this.peers).filter(
       (peer) => peer.role === "server" && peer.id !== excludePeerId,
     );
+    return this.serverPeerSelector
+      ? this.serverPeerSelector(id, serverPeers)
+      : serverPeers;
   }
 
   handleSyncMessage(msg: SyncMessage, peer: PeerState) {
@@ -221,6 +226,23 @@ export class SyncManager {
     id: RawCoID,
     peer: PeerState,
     seen: Set<RawCoID> = new Set(),
+  ) {
+    this.sendNewContent(id, peer, seen, true);
+  }
+
+  sendNewContentWithoutDependencies(
+    id: RawCoID,
+    peer: PeerState,
+    seen: Set<RawCoID> = new Set(),
+  ) {
+    this.sendNewContent(id, peer, seen, false);
+  }
+
+  private sendNewContent(
+    id: RawCoID,
+    peer: PeerState,
+    seen: Set<RawCoID> = new Set(),
+    includeDependencies: boolean,
   ) {
     if (seen.has(id)) {
       return;
@@ -234,8 +256,10 @@ export class SyncManager {
       return;
     }
 
-    for (const dependency of coValue.getDependedOnCoValues()) {
-      this.sendNewContentIncludingDependencies(dependency, peer, seen);
+    if (includeDependencies) {
+      for (const dependency of coValue.getDependedOnCoValues()) {
+        this.sendNewContentIncludingDependencies(dependency, peer, seen);
+      }
     }
 
     const newContentPieces = coValue.verified.newContentSince(
@@ -402,7 +426,7 @@ export class SyncManager {
       return;
     }
 
-    const peers = this.getServerPeers(peer.id);
+    const peers = this.getServerPeers(msg.id, peer.id);
 
     coValue.load(peers);
 
@@ -442,7 +466,11 @@ export class SyncManager {
     }
 
     if (coValue.isAvailable()) {
-      this.sendNewContentIncludingDependencies(msg.id, peer);
+      if (peer.role === "server") {
+        this.sendNewContentWithoutDependencies(msg.id, peer);
+      } else {
+        this.sendNewContentIncludingDependencies(msg.id, peer);
+      }
     }
   }
 
@@ -524,7 +552,7 @@ export class SyncManager {
           if (!dependencyCoValue.hasVerifiedContent()) {
             coValue.markMissingDependency(dependency);
 
-            const peers = this.getServerPeers();
+            const peers = this.getServerPeers(dependencyCoValue.id);
 
             // if the peer that sent the content is a client, we add it to the list of peers
             // to also ask them for the dependency
@@ -607,7 +635,7 @@ export class SyncManager {
             // This covers the case where we are getting a new session on an already loaded coValue
             // where we need to load the account to get their public key
             if (!coValue.missingDependencies.has(accountId)) {
-              const peers = this.getServerPeers();
+              const peers = this.getServerPeers(account.id);
 
               if (peer?.role === "client") {
                 // if the peer that sent the content is a client, we add it to the list of peers
@@ -727,7 +755,7 @@ export class SyncManager {
       this.storeContent(contentToStore);
     }
 
-    for (const peer of this.peersInPriorityOrder()) {
+    for (const peer of this.getPeers(coValue.id)) {
       /**
        * We sync the content against the source peer if it is a client or server peers
        * to upload any content that is available on the current node and not on the source peer.
@@ -771,26 +799,12 @@ export class SyncManager {
     return this.sendNewContentIncludingDependencies(msg.id, peer);
   }
 
-  dirtyCoValuesTrackingSets: Set<Set<RawCoID>> = new Set();
-  trackDirtyCoValues() {
-    const trackingSet = new Set<RawCoID>();
-
-    this.dirtyCoValuesTrackingSets.add(trackingSet);
-
-    return {
-      done: () => {
-        this.dirtyCoValuesTrackingSets.delete(trackingSet);
-
-        return trackingSet;
-      },
-    };
-  }
-
   private syncQueue = new LocalTransactionsSyncQueue((content) =>
     this.syncContent(content),
   );
   syncHeader = this.syncQueue.syncHeader;
   syncLocalTransaction = this.syncQueue.syncTransaction;
+  trackDirtyCoValues = this.syncQueue.trackDirtyCoValues;
 
   syncContent(content: NewContentMessage) {
     const coValue = this.local.getCoValue(content.id);
@@ -799,7 +813,7 @@ export class SyncManager {
 
     const contentKnownState = knownStateFromContent(content);
 
-    for (const peer of this.peersInPriorityOrder()) {
+    for (const peer of this.getPeers(coValue.id)) {
       if (peer.closed) continue;
       if (coValue.isErroredInPeer(peer.id)) continue;
 
@@ -818,7 +832,7 @@ export class SyncManager {
       peer.trackToldKnownState(coValue.id);
     }
 
-    for (const peer of this.getPeers()) {
+    for (const peer of this.getPeers(coValue.id)) {
       this.syncState.triggerUpdate(peer.id, coValue.id);
     }
   }
@@ -899,7 +913,7 @@ export class SyncManager {
   }
 
   waitForSync(id: RawCoID, timeout = 60_000) {
-    const peers = this.getPeers();
+    const peers = this.getPeers(id);
 
     return Promise.all(
       peers

package/src/tests/PureJSCrypto.test.ts

@@ -126,6 +126,72 @@ describe("PureJSCrypto", () => {
 
     expect(map.get("count")).toEqual(0);
   });
+
+  it("can add a meta to a private transaction", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.core.makeTransaction([], "private", {
+      meta: {
+        count: 1,
+      },
+    });
+
+    await map.core.waitForSync();
+
+    const session2 = client.spawnNewSession();
+
+    const mapInOtherSession = await loadCoValueOrFail(session2.node, map.id);
+
+    const decryptedMeta =
+      mapInOtherSession.core.verified.decryptTransactionMeta(
+        client.node.currentSessionID,
+        0,
+        map.core.getCurrentReadKey().secret!,
+      );
+
+    expect(decryptedMeta).toEqual({
+      meta: {
+        count: 1,
+      },
+    });
+  });
+
+  it("can add a meta to a trusting transaction", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.core.makeTransaction([], "trusting", {
+      meta: {
+        count: 1,
+      },
+    });
+
+    await map.core.waitForSync();
+
+    const session2 = client.spawnNewSession();
+
+    const mapInOtherSession = await loadCoValueOrFail(session2.node, map.id);
+
+    const transferredMeta = JSON.parse(
+      mapInOtherSession.core.verified.sessions.get(client.node.currentSessionID)
+        ?.transactions[0]?.meta!,
+    );
+
+    expect(transferredMeta).toEqual({
+      meta: {
+        count: 1,
+      },
+    });
+  });
 });
 
 describe("PureJSSessionLog", () => {

package/src/tests/WasmCrypto.test.ts

@@ -125,4 +125,92 @@ describe("WasmCrypto", () => {
 
     expect(map.get("count")).toEqual(0);
   });
+
+  it("can add a meta to a private transaction", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.core.makeTransaction([], "private", {
+      meta: {
+        count: 1,
+      },
+    });
+
+    await map.core.waitForSync();
+
+    const session2 = client.spawnNewSession();
+
+    const mapInOtherSession = await loadCoValueOrFail(session2.node, map.id);
+
+    const decryptedMeta =
+      mapInOtherSession.core.verified.decryptTransactionMeta(
+        client.node.currentSessionID,
+        0,
+        mapInOtherSession.core.getCurrentReadKey().secret!,
+      );
+
+    expect(decryptedMeta).toEqual({
+      meta: {
+        count: 1,
+      },
+    });
+  });
+
+  it("can add a meta to a trusting transaction", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
+
+    const group = client.node.createGroup();
+    const map = group.createMap();
+
+    map.core.makeTransaction([], "trusting", {
+      meta: {
+        count: 1,
+      },
+    });
+
+    await map.core.waitForSync();
+
+    const session2 = client.spawnNewSession();
+
+    const mapInOtherSession = await loadCoValueOrFail(session2.node, map.id);
+
+    const transferredMeta = JSON.parse(
+      mapInOtherSession.core.verified.sessions.get(client.node.currentSessionID)
+        ?.transactions[0]?.meta!,
+    );
+
+    expect(transferredMeta).toEqual({
+      meta: {
+        count: 1,
+      },
+    });
+  });
+
+  it("fails to verify signatures without a signer ID", async () => {
+    const agentSecret = wasmCrypto.newRandomAgentSecret();
+    const sessionID = wasmCrypto.newRandomSessionID(
+      wasmCrypto.getAgentID(agentSecret),
+    );
+
+    const sessionLog = wasmCrypto.createSessionLog("co_z12345678", sessionID);
+    expect(() =>
+      sessionLog.tryAdd(
+        [
+          {
+            privacy: "trusting",
+            changes: stableStringify([{ op: "set", key: "count", value: 1 }]),
+            madeAt: Date.now(),
+          },
+        ],
+        "signature_z12345678",
+        false,
+      ),
+    ).toThrow(expect.stringContaining("Signature verification failed"));
+  });
 });

package/src/tests/coList.test.ts

@@ -409,3 +409,22 @@ test("totalValidTransactions should return the number of valid transactions proc
     listOnOtherClient.core.getCurrentContent().totalValidTransactions,
   ).toEqual(2);
 });
+
+test("Should ignore unknown meta transactions", () => {
+  const node = nodeWithRandomAgentAndSessionID();
+
+  const coValue = node.createCoValue({
+    type: "colist",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  coValue.makeTransaction([], "trusting", { unknownMeta: 1 });
+
+  const content = expectList(coValue.getCurrentContent());
+
+  content.append("first", 0, "trusting");
+
+  expect(content.toJSON()).toEqual(["first"]);
+});

package/src/tests/coMap.test.ts

@@ -217,6 +217,27 @@ test("Can set items in bulk with assign", () => {
   });
 });
 
+test("Should ignore unknown meta transactions", () => {
+  const node = nodeWithRandomAgentAndSessionID();
+
+  const coValue = node.createCoValue({
+    type: "comap",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  coValue.makeTransaction([], "trusting", { unknownMeta: 1 });
+
+  const content = expectMap(coValue.getCurrentContent());
+
+  expect(content.type).toEqual("comap");
+
+  content.set("key1", "set1", "trusting");
+
+  expect(content.get("key1")).toEqual("set1");
+});
+
 test("totalValidTransactions should return the number of valid transactions processed", async () => {
   const client = setupTestNode({
     connected: true,

package/src/tests/coPlainText.test.ts

@@ -298,6 +298,24 @@ test("Splits into and from grapheme string arrays", () => {
   expect(text).toEqual("👋 안녕!");
 });
 
+test("Should ignore unknown meta transactions", () => {
+  const node = nodeWithRandomAgentAndSessionID();
+
+  const coValue = node.createCoValue({
+    type: "coplaintext",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: null,
+    ...Crypto.createdNowUnique(),
+  });
+
+  coValue.makeTransaction([], "trusting", { unknownMeta: 1 });
+
+  const content = expectPlainText(coValue.getCurrentContent());
+
+  content.insertAfter(0, "hello", "trusting");
+  expect(content.toString()).toEqual("hello");
+});
+
 test("chunks transactions when when the chars are longer than MAX_RECOMMENDED_TX_SIZE", async () => {
   setMaxRecommendedTxSize(5);
 

package/src/tests/coStream.test.ts

@@ -127,6 +127,42 @@ test("Can push into RawBinaryCoStream", () => {
   });
 });
 
+test("Should ignore meta transactions on RawBinaryCoStream", () => {
+  const node = nodeWithRandomAgentAndSessionID();
+
+  const coValue = node.createCoValue({
+    type: "costream",
+    ruleset: { type: "unsafeAllowAll" },
+    meta: { type: "binary" },
+    ...Crypto.createdNowUnique(),
+  });
+
+  coValue.makeTransaction([], "trusting", { unknownMeta: 1 });
+
+  const content = coValue.getCurrentContent();
+
+  if (
+    content.type !== "costream" ||
+    content.headerMeta?.type !== "binary" ||
+    !(content instanceof RawBinaryCoStream)
+  ) {
+    throw new Error("Expected binary stream");
+  }
+
+  content.startBinaryStream(
+    { mimeType: "text/plain", fileName: "test.txt" },
+    "trusting",
+  );
+  content.pushBinaryStreamChunk(new Uint8Array([1, 2, 3]), "trusting");
+  content.endBinaryStream("trusting");
+  expect(content.getBinaryChunks()).toEqual({
+    mimeType: "text/plain",
+    fileName: "test.txt",
+    chunks: [new Uint8Array([1, 2, 3])],
+    finished: true,
+  });
+});
+
 test("When adding large transactions (small fraction of MAX_RECOMMENDED_TX_SIZE), we store an inbetween signature every time we reach MAX_RECOMMENDED_TX_SIZE and split up newContentSince accordingly", () => {
   const node = nodeWithRandomAgentAndSessionID();
 

package/src/tests/coValueCore.test.ts

@@ -20,9 +20,11 @@ import {
   loadCoValueOrFail,
   nodeWithRandomAgentAndSessionID,
   randomAgentAndSessionID,
+  setupTestNode,
   tearDownTestMetricReader,
 } from "./testUtils.js";
 import { CO_VALUE_PRIORITY } from "../priority.js";
+import { determineValidTransactions } from "../permissions.js";
 
 const Crypto = await WasmCrypto.create();
 
@@ -32,6 +34,7 @@ const agentSecret =
 
 beforeEach(() => {
   metricReader = createTestMetricReader();
+  setupTestNode({ isSyncServer: true });
 });
 
 afterEach(() => {
@@ -53,6 +56,7 @@ test("transactions with wrong signature are rejected", () => {
       node.currentSessionID,
       node.getCurrentAgent(),
       [{ hello: "world" }],
+      undefined,
     );
 
   transaction.madeAt = Date.now() + 1000;
@@ -264,6 +268,51 @@ test("listeners are notified even if the previous listener threw an error", asyn
   errorLog.mockRestore();
 });
 
+test("creates a transaction with trusting meta information", async () => {
+  const client = setupTestNode();
+
+  const group = client.node.createGroup();
+  const map = group.createMap();
+  map.core.makeTransaction([], "trusting", {
+    meta: true,
+  });
+
+  const validTransactions = determineValidTransactions(map.core);
+
+  expect(validTransactions[0]?.tx.meta).toBe(`{"meta":true}`);
+});
+
+test("creates a transaction with private meta information", async () => {
+  const client = setupTestNode({ connected: true });
+
+  const group = client.node.createGroup();
+  const map = group.createMap();
+  map.core.makeTransaction([], "private", {
+    meta: true,
+  });
+
+  const localTransactionMeta = map.core.verified.decryptTransactionMeta(
+    client.node.currentSessionID,
+    0,
+    map.core.getCurrentReadKey().secret!,
+  );
+
+  expect(localTransactionMeta).toEqual({ meta: true });
+
+  const newSession = client.spawnNewSession();
+
+  const mapOnNewSession = await loadCoValueOrFail(newSession.node, map.id);
+
+  const syncedTransactionMeta =
+    mapOnNewSession.core.verified.decryptTransactionMeta(
+      client.node.currentSessionID,
+      0,
+      mapOnNewSession.core.getCurrentReadKey().secret!,
+    );
+
+  expect(syncedTransactionMeta).toEqual({ meta: true });
+});
+
 test("getValidTransactions should skip private transactions with invalid JSON", () => {
   const [agent, sessionID] = agentAndSessionIDFromSecret(agentSecret);
   const node = new LocalNode(agent.agentSecret, sessionID, Crypto);

package/src/tests/coreWasm.test.ts

@@ -85,6 +85,7 @@ describe("SessionLog WASM", () => {
       key.id,
       key.secret,
       0,
+      undefined,
     );
 
     expect(signature).toMatch(/^signature_z[a-zA-Z0-9]+$/);

package/src/tests/sync.known.test.ts

@@ -0,0 +1,109 @@
+import { beforeEach, describe, expect, test } from "vitest";
+import { setCoValueLoadingRetryDelay } from "../config";
+import {
+  SyncMessagesLog,
+  TEST_NODE_CONFIG,
+  blockMessageTypeOnOutgoingPeer,
+  getSyncServerConnectedPeer,
+  loadCoValueOrFail,
+  setupTestAccount,
+  setupTestNode,
+} from "./testUtils";
+
+let jazzCloud: ReturnType<typeof setupTestNode>;
+
+// Set a short timeout to make the tests on unavailable complete faster
+setCoValueLoadingRetryDelay(100);
+
+beforeEach(async () => {
+  // We want to simulate a real world communication that happens asynchronously
+  TEST_NODE_CONFIG.withAsyncPeers = true;
+
+  SyncMessagesLog.clear();
+  jazzCloud = setupTestNode({ isSyncServer: true });
+});
+
+describe("sending known coValues", () => {
+  test("dependencies are included when responding to a client", async () => {
+    const group = jazzCloud.node.createGroup();
+    const map = group.createMap();
+    map.set("hello", "world", "trusting");
+
+    const { node: client } = setupTestNode({
+      connected: true,
+    });
+
+    map.set("hello2", "world2", "trusting");
+    await map.core.waitForSync();
+
+    const mapOnClient = await loadCoValueOrFail(client, map.id);
+    expect(mapOnClient.get("hello")).toEqual("world");
+    expect(mapOnClient.get("hello2")).toEqual("world2");
+
+    expect(
+      SyncMessagesLog.getMessages({
+        Group: group.core,
+        Map: map.core,
+      }),
+    ).toMatchInlineSnapshot(`
+      [
+        "client -> server | LOAD Map sessions: empty",
+        "server -> client | CONTENT Group header: true new: After: 0 New: 3",
+        "server -> client | CONTENT Map header: true new: After: 0 New: 2",
+        "client -> server | KNOWN Group sessions: header/3",
+        "client -> server | KNOWN Map sessions: header/2",
+      ]
+    `);
+  });
+
+  test("dependencies are excluded when responding to a server", async () => {
+    // Create a disconnected client
+    const { node: client, accountID } = await setupTestAccount({
+      connected: false,
+    });
+    const account = client.expectCurrentAccount(accountID);
+
+    // Prepare a group -- this will be a non-account dependency of a forthcoming map.
+    const group = client.createGroup();
+    group.addMember("everyone", "writer");
+
+    // Let the queue drain
+    await new Promise((resolve) => setTimeout(resolve, 1));
+
+    // Disable transaction verification on the server so it doesn't ask for dependencies.
+    jazzCloud.node.syncManager.disableTransactionVerification();
+
+    // Connect the client, but don't setup syncing just yet...
+    const { peer } = getSyncServerConnectedPeer({
+      peerId: client.getCurrentAgent().id,
+      syncServer: jazzCloud.node,
+    });
+
+    // Disable reconciliation while we setup syncing because we don't want the
+    // server to know about our forthcoming map's dependencies (group + account).
+    const blocker = blockMessageTypeOnOutgoingPeer(peer, "load", {});
+    client.syncManager.addPeer(peer);
+    blocker.unblock();
+
+    // Create a map and set a value on it, this will trigger:
+    //   - CONTENT from client to server
+    //   - KNOWN from server to client
+    //
+    // We don't expect any more messages to be sent from client to server in this
+    // case because clients shouldn't greedily send dependencies to a server.
+    const map = group.createMap();
+    await map.core.waitForSync();
+
+    const syncMessages = SyncMessagesLog.getMessages({
+      Account: account.core,
+      Group: group.core,
+      Map: map.core,
+    });
+    expect(
+      syncMessages.some(
+        (msg) =>
+          msg.includes("CONTENT Account") || msg.includes("CONTENT Group"),
+      ),
+    ).toBe(false);
+  });
+});