cojson 0.18.37 → 0.19.0

package/src/permissions.ts

@@ -18,7 +18,6 @@ import {
   getParentGroupId,
 } from "./ids.js";
 import { JsonValue } from "./jsonValue.js";
-import { logger } from "./logger.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
 export type PermissionsDef =
@@ -71,23 +70,6 @@ function canAdmin(role: Role | undefined): boolean {
   return role === "admin" || role === "manager";
 }
 
-let logPermissionErrors = true;
-
-export function disablePermissionErrors() {
-  logPermissionErrors = false;
-}
-
-function logPermissionError(
-  message: string,
-  attributes?: Record<string, JsonValue>,
-) {
-  if (logPermissionErrors === false) {
-    return;
-  }
-
-  logger.debug("Permission error: " + message, attributes);
-}
-
 export function determineValidTransactions(coValue: CoValueCore): void {
   if (!coValue.isAvailable()) {
     throw new Error("determineValidTransactions CoValue is not available");
@@ -129,7 +111,10 @@ export function determineValidTransactions(coValue: CoValueCore): void {
       );
 
       if (!effectiveTransactor) {
-        tx.markInvalid();
+        tx.markInvalid("Transactor not found in group", {
+          transactor: tx.author,
+          group: groupAtTime.toJSON(),
+        });
         continue;
       }
 
@@ -157,7 +142,10 @@ export function determineValidTransactions(coValue: CoValueCore): void {
         transactorRoleAtTxTime !== "writer" &&
         transactorRoleAtTxTime !== "writeOnly"
       ) {
-        tx.markInvalid();
+        tx.markInvalid("Transactor has no write permissions", {
+          transactor: tx.author,
+          transactorRole: transactorRoleAtTxTime ?? "undefined",
+        });
         continue;
       }
 
@@ -263,10 +251,9 @@ function determineValidTransactionsForGroup(
         transaction.markValid();
         continue;
       } else {
-        logPermissionError(
+        transaction.markInvalid(
           "Only admins can make private transactions in groups",
         );
-        transaction.markInvalid();
         continue;
       }
     }
@@ -286,21 +273,18 @@ function determineValidTransactionsForGroup(
       | MapOpPayload<`child_${CoID<RawGroup>}`, CoID<RawGroup>>;
 
     if (changes.length !== 1) {
-      logPermissionError("Group transaction must have exactly one change");
-      transaction.markInvalid();
+      transaction.markInvalid("Group transaction must have exactly one change");
       continue;
     }
 
     if (change.op !== "set") {
-      logPermissionError("Group transaction must set a role or readKey");
-      transaction.markInvalid();
+      transaction.markInvalid("Group transaction must set a role or readKey");
       continue;
     }
 
     if (change.key === "readKey") {
       if (!canAdmin(transactorRole)) {
-        logPermissionError("Only admins can set readKeys");
-        transaction.markInvalid();
+        transaction.markInvalid("Only admins can set readKeys");
         continue;
       }
 
@@ -308,8 +292,7 @@ function determineValidTransactionsForGroup(
       continue;
     } else if (change.key === "profile") {
       if (!canAdmin(transactorRole)) {
-        logPermissionError("Only admins can set profile");
-        transaction.markInvalid();
+        transaction.markInvalid("Only admins can set profile");
         continue;
       }
 
@@ -317,7 +300,7 @@ function determineValidTransactionsForGroup(
       continue;
     } else if (change.key === "root") {
       if (!canAdmin(transactorRole)) {
-        logPermissionError("Only admins can set root");
+        transaction.markInvalid("Only admins can set root");
         continue;
       }
 
@@ -337,8 +320,7 @@ function determineValidTransactionsForGroup(
         transactorRole !== "writeOnlyInvite" &&
         !isOwnWriteKeyRevelation(change.key, transactor, writeOnlyKeys)
       ) {
-        logPermissionError("Only admins and managers can reveal keys");
-        transaction.markInvalid();
+        transaction.markInvalid("Only admins and managers can reveal keys");
         continue;
       }
 
@@ -346,10 +328,9 @@ function determineValidTransactionsForGroup(
       continue;
     } else if (isParentExtension(change.key)) {
       if (!canAdmin(transactorRole)) {
-        logPermissionError(
+        transaction.markInvalid(
           "Only admins and managers can set parent extensions",
         );
-        transaction.markInvalid();
         continue;
       }
 
@@ -361,16 +342,14 @@ function determineValidTransactionsForGroup(
       );
 
       if (!parentGroupCore.isGroup()) {
-        logPermissionError("Parent group is not a group");
-        transaction.markInvalid();
+        transaction.markInvalid("Parent group is not a group");
         continue;
       }
 
       const parentGroup = expectGroup(parentGroupCore.getCurrentContent());
 
       if (isSelfExtension(coValue, parentGroup)) {
-        logPermissionError("Parent group is a circular dependency");
-        transaction.markInvalid();
+        transaction.markInvalid("Parent group is a circular dependency");
         continue;
       }
 
@@ -385,8 +364,7 @@ function determineValidTransactionsForGroup(
       transaction.markValid();
       continue;
     } else if (isChildExtension(change.key)) {
-      logPermissionError("Child extensions are not allowed anymore");
-      transaction.markInvalid();
+      transaction.markInvalid("Child extensions are not allowed anymore");
       continue;
     } else if (isWriteKeyForMember(change.key)) {
       const memberKey = getAccountOrAgentFromWriteKeyForMember(change.key);
@@ -397,8 +375,7 @@ function determineValidTransactionsForGroup(
         transactorRole !== "writeOnlyInvite" &&
         memberKey !== transactor
       ) {
-        logPermissionError("Only admins and managers can set writeKeys");
-        transaction.markInvalid();
+        transaction.markInvalid("Only admins and managers can set writeKeys");
         continue;
       }
 
@@ -413,10 +390,9 @@ function determineValidTransactionsForGroup(
        * blocking them from accessing the group.ß
        */
       if (writeKeys.has(change.key) && !canAdmin(transactorRole)) {
-        logPermissionError(
+        transaction.markInvalid(
           "Write key already exists and can't be overridden by invite",
         );
-        transaction.markInvalid();
         continue;
       }
 
@@ -442,8 +418,7 @@ function determineValidTransactionsForGroup(
       assignedRole !== "readerInvite" &&
       assignedRole !== "writeOnlyInvite"
     ) {
-      logPermissionError("Group transaction must set a valid role");
-      transaction.markInvalid();
+      transaction.markInvalid("Group transaction must set a valid role");
       continue;
     }
 
@@ -456,10 +431,9 @@ function determineValidTransactionsForGroup(
         assignedRole === "revoked"
       )
     ) {
-      logPermissionError(
+      transaction.markInvalid(
         "Everyone can only be set to reader, writer, writeOnly or revoked",
       );
-      transaction.markInvalid();
       continue;
     }
 
@@ -474,11 +448,6 @@ function determineValidTransactionsForGroup(
       transaction.markValid();
     }
 
-    function markTransactionAsInvalid(message: string) {
-      logPermissionError(message);
-      transaction.markInvalid();
-    }
-
     // is first self promotion to admin
     if (
       transactorRole === undefined &&
@@ -511,7 +480,7 @@ function determineValidTransactionsForGroup(
         assignedRole !== "admin" &&
         affectedMember !== transactor
       ) {
-        markTransactionAsInvalid("Admins can't demote admins.");
+        transaction.markInvalid("Admins can't demote admins.");
         continue;
       }
 
@@ -527,20 +496,20 @@ function determineValidTransactionsForGroup(
      */
     if (transactorRole === "manager") {
       if (affectedMemberRole === "admin") {
-        markTransactionAsInvalid("Managers can't demote admins.");
+        transaction.markInvalid("Managers can't demote admins.");
         continue;
       }
       if (change.value === "admin") {
-        markTransactionAsInvalid("Managers can't promote to admin.");
+        transaction.markInvalid("Managers can't promote to admin.");
         continue;
       }
 
       if (change.value === "adminInvite") {
-        markTransactionAsInvalid("Managers can't invite admins.");
+        transaction.markInvalid("Managers can't invite admins.");
         continue;
       }
       if (change.value === "managerInvite") {
-        markTransactionAsInvalid("Managers can't invite managers.");
+        transaction.markInvalid("Managers can't invite managers.");
         continue;
       }
 
@@ -550,32 +519,31 @@ function determineValidTransactionsForGroup(
 
     if (transactorRole === "adminInvite") {
       if (change.value !== "admin") {
-        logPermissionError("AdminInvites can only create admins.");
-        transaction.markInvalid();
+        transaction.markInvalid("AdminInvites can only create admins.");
         continue;
       }
     } else if (transactorRole === "managerInvite") {
       if (change.value !== "manager") {
-        markTransactionAsInvalid("managerInvite can only create managers.");
+        transaction.markInvalid("managerInvite can only create managers.");
         continue;
       }
     } else if (transactorRole === "writerInvite") {
       if (change.value !== "writer") {
-        markTransactionAsInvalid("WriterInvites can only create writers.");
+        transaction.markInvalid("WriterInvites can only create writers.");
         continue;
       }
     } else if (transactorRole === "readerInvite") {
       if (change.value !== "reader") {
-        markTransactionAsInvalid("ReaderInvites can only create reader.");
+        transaction.markInvalid("ReaderInvites can only create reader.");
         continue;
       }
     } else if (transactorRole === "writeOnlyInvite") {
       if (change.value !== "writeOnly") {
-        markTransactionAsInvalid("WriteOnlyInvites can only create writeOnly.");
+        transaction.markInvalid("WriteOnlyInvites can only create writeOnly.");
         continue;
       }
     } else {
-      markTransactionAsInvalid(
+      transaction.markInvalid(
         "Group transaction must be made by current admin, manager, or invite",
       );
       continue;

package/src/queue/LocalTransactionsSyncQueue.ts

@@ -50,33 +50,14 @@ export class LocalTransactionsSyncQueue {
     coValue: VerifiedState,
     knownStateBefore: CoValueKnownState,
   ) {
-    const content = coValue.newContentSince(knownStateBefore, {
-      skipExpectContentUntil: true, // we need to calculate the streaming header considering the current batch
-    });
+    const content = coValue.newContentSince(knownStateBefore);
 
     if (!content) {
       return;
     }
 
-    let firstChunk = this.firstChunks.get(coValue.id);
-
     for (const piece of content) {
       this.batch.push(piece);
-
-      // Check if the local content updates are in streaming, if so we need to add the info to the first chunk
-      if (firstChunk) {
-        if (!firstChunk.expectContentUntil) {
-          firstChunk.expectContentUntil =
-            knownStateFromContent(firstChunk).sessions;
-        }
-        combineKnownStateSessions(
-          firstChunk.expectContentUntil,
-          knownStateFromContent(piece).sessions,
-        );
-      } else {
-        firstChunk = piece;
-        this.firstChunks.set(coValue.id, firstChunk);
-      }
     }
   }
 

package/src/sync.ts

@@ -484,6 +484,15 @@ export class SyncManager {
           ? "import"
           : peer?.role;
 
+    // TODO: We can't handle client-to-client streaming until we
+    // handle the streaming state reset on disconnection
+    if (peer?.role === "client" && msg.expectContentUntil) {
+      msg = {
+        ...msg,
+        expectContentUntil: undefined,
+      };
+    }
+
     coValue.addDependenciesFromContentMessage(msg);
 
     // If some of the dependencies are missing, we wait for them to be available

package/src/tests/StorageApiAsync.test.ts

@@ -539,16 +539,16 @@ describe("StorageApiAsync", () => {
         }),
       ).toMatchInlineSnapshot(`
         [
-          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
-          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
+          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
           "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
           "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
           "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
           "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
           "test-storage -> test | KNOWN CORRECTION Core2 sessions: empty",
           "test -> test-storage | CONTENT Core2 header: true new: After: 0 New: 3",
-          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
-          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
+          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
           "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
           "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
         ]
@@ -606,14 +606,14 @@ describe("StorageApiAsync", () => {
         }),
       ).toMatchInlineSnapshot(`
         [
-          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1 expectContentUntil: header/3",
-          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1 expectContentUntil: header/3",
+          "test -> test-storage | CONTENT Core header: false new: After: 1 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 1 New: 1",
           "test -> test-storage | CONTENT Core header: false new: After: 2 New: 1",
           "test -> test-storage | CONTENT Core2 header: false new: After: 2 New: 1",
           "test-storage -> test | KNOWN CORRECTION Core sessions: empty",
           "test -> test-storage | CONTENT Core header: true new: After: 0 New: 3",
-          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1 expectContentUntil: header/5",
-          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1 expectContentUntil: header/5",
+          "test -> test-storage | CONTENT Core header: false new: After: 3 New: 1",
+          "test -> test-storage | CONTENT Core2 header: false new: After: 3 New: 1",
           "test -> test-storage | CONTENT Core header: false new: After: 4 New: 1",
           "test -> test-storage | CONTENT Core2 header: false new: After: 4 New: 1",
         ]

package/src/tests/coValueCore.isStreaming.test.ts

@@ -2,6 +2,7 @@ import { assert, beforeEach, describe, expect, test } from "vitest";
 import {
   SyncMessagesLog,
   TEST_NODE_CONFIG,
+  fillCoMapWithLargeData,
   loadCoValueOrFail,
   setupTestAccount,
   setupTestNode,
@@ -49,19 +50,7 @@ describe("isStreaming", () => {
     await group.core.waitForSync();
     client.disconnect();
 
-    const map = group.createMap();
-
-    // Generate a large amount of data that requires multiple chunks
-    const dataSize = 1 * 1024 * 100;
-    const chunkSize = 1024; // 1KB chunks
-    const chunks = dataSize / chunkSize;
-
-    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
-
-    for (let i = 0; i < chunks; i++) {
-      const key = `key${i}`;
-      map.set(key, value, "trusting");
-    }
+    const map = fillCoMapWithLargeData(group.createMap());
 
     const newSession = client.spawnNewSession();
 
@@ -101,16 +90,7 @@ describe("isStreaming", () => {
     client.disconnect();
 
     // Generate a large amount of data that requires multiple chunks
-    const dataSize = 1 * 1024 * 100;
-    const chunkSize = 1024; // 1KB chunks
-    const chunks = dataSize / chunkSize;
-
-    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
-
-    for (let i = 0; i < chunks; i++) {
-      const key = `key${i}`;
-      map.set(key, value, "trusting");
-    }
+    fillCoMapWithLargeData(map);
 
     const newSession = client.spawnNewSession();
 
@@ -134,41 +114,81 @@ describe("isStreaming", () => {
     expect(mapInNewSession.core.isStreaming()).toBe(false);
   });
 
-  test("streaming a large value between two clients should be streaming until all chunks are sent", async () => {
-    const client = setupTestNode();
-    client.connectToSyncServer({
-      ourName: "initialClient",
-    });
-    const streamingClient = client.spawnNewSession();
-    streamingClient.connectToSyncServer({
-      ourName: "streamingClient",
-    });
+  // TODO: We can't handle client-to-client streaming until we
+  // handle the streaming state reset on disconnection
+  // Otherwise the other client might wait for a content that will never be sent
+  test.fails(
+    "streaming a large value between two clients should be streaming until all chunks are sent",
+    async () => {
+      const client = setupTestNode();
+      client.connectToSyncServer({
+        ourName: "initialClient",
+      });
+      const streamingClient = client.spawnNewSession();
+      streamingClient.connectToSyncServer({
+        ourName: "streamingClient",
+      });
+
+      const group = client.node.createGroup();
+
+      await group.core.waitForSync();
+      client.disconnect();
+
+      const map = fillCoMapWithLargeData(group.createMap());
+
+      const loadingClient = client.spawnNewSession();
+      loadingClient.connectToSyncServer({
+        ourName: "loadingClient",
+      });
+
+      await loadCoValueOrFail(loadingClient.node, group.id);
+
+      const content = map.core.verified.newContentSince(undefined);
+      assert(content);
+      const lastChunk = content.pop();
+      assert(lastChunk);
+
+      for (const chunk of content) {
+        streamingClient.node.syncManager.handleNewContent(chunk, "import");
+      }
+
+      await streamingClient.node.syncManager.waitForAllCoValuesSync();
+
+      const mapInLoadingClient = await loadCoValueOrFail(
+        loadingClient.node,
+        map.id,
+      );
 
-    const group = client.node.createGroup();
+      expect(mapInLoadingClient.core.isStreaming()).toBe(true);
 
-    await group.core.waitForSync();
-    client.disconnect();
+      streamingClient.node.syncManager.handleNewContent(lastChunk, "import");
 
-    const map = group.createMap();
+      await waitFor(() => {
+        expect(mapInLoadingClient.core.knownState()).toEqual(
+          map.core.knownState(),
+        );
+      });
 
-    // Generate a large amount of data that requires multiple chunks
-    const dataSize = 1 * 1024 * 100;
-    const chunkSize = 1024; // 1KB chunks
-    const chunks = dataSize / chunkSize;
+      expect(mapInLoadingClient.core.isStreaming()).toBe(false);
+    },
+  );
 
-    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
+  test("should be false when getting streaming content that is already in the known state", async () => {
+    const client = setupTestNode({
+      connected: true,
+    });
 
-    for (let i = 0; i < chunks; i++) {
-      const key = `key${i}`;
-      map.set(key, value, "trusting");
-    }
+    const group = client.node.createGroup();
 
-    const loadingClient = client.spawnNewSession();
-    loadingClient.connectToSyncServer({
-      ourName: "loadingClient",
-    });
+    await group.core.waitForSync();
+
+    const map = fillCoMapWithLargeData(group.createMap());
 
-    await loadCoValueOrFail(loadingClient.node, group.id);
+    await map.core.waitForSync();
+    const newSession = client.spawnNewSession();
+
+    const mapInNewSession1 = await loadCoValueOrFail(newSession.node, map.id);
+    await mapInNewSession1.core.waitForFullStreaming();
 
     const content = map.core.verified.newContentSince(undefined);
     assert(content);
@@ -176,57 +196,28 @@ describe("isStreaming", () => {
     assert(lastChunk);
 
     for (const chunk of content) {
-      streamingClient.node.syncManager.handleNewContent(chunk, "import");
+      newSession.node.syncManager.handleNewContent(chunk, "import");
     }
 
-    await streamingClient.node.syncManager.waitForAllCoValuesSync();
-
-    const mapInLoadingClient = await loadCoValueOrFail(
-      loadingClient.node,
-      map.id,
-    );
-
-    expect(mapInLoadingClient.core.isStreaming()).toBe(true);
-
-    streamingClient.node.syncManager.handleNewContent(lastChunk, "import");
-
-    await waitFor(() => {
-      expect(mapInLoadingClient.core.knownState()).toEqual(
-        map.core.knownState(),
-      );
-    });
+    const mapInNewSession = await loadCoValueOrFail(newSession.node, map.id);
 
-    expect(mapInLoadingClient.core.isStreaming()).toBe(false);
+    expect(mapInNewSession.core.isStreaming()).toBe(false);
   });
 
-  test("should be false when getting streaming content that is already in the known state", async () => {
-    const client = setupTestNode({
-      connected: true,
-    });
+  test("streaming state from clients should be ignored", async () => {
+    const client = setupTestNode();
 
     const group = client.node.createGroup();
 
     await group.core.waitForSync();
 
-    const map = group.createMap();
-
-    // Generate a large amount of data that requires multiple chunks
-    const dataSize = 1 * 1024 * 100;
-    const chunkSize = 1024; // 1KB chunks
-    const chunks = dataSize / chunkSize;
-
-    const value = Buffer.alloc(chunkSize, `value$`).toString("base64");
-
-    for (let i = 0; i < chunks; i++) {
-      const key = `key${i}`;
-      map.set(key, value, "trusting");
-    }
+    const map = fillCoMapWithLargeData(group.createMap());
 
     await map.core.waitForSync();
     const newSession = client.spawnNewSession();
-
-    const mapInNewSession1 = await loadCoValueOrFail(newSession.node, map.id);
-    await mapInNewSession1.core.waitForFullStreaming();
+    newSession.connectToSyncServer({
+      ourName: "streamingClient",
+    });
 
     const content = map.core.verified.newContentSince(undefined);
     assert(content);
@@ -237,9 +228,11 @@ describe("isStreaming", () => {
       newSession.node.syncManager.handleNewContent(chunk, "import");
     }
 
-    const mapInNewSession = await loadCoValueOrFail(newSession.node, map.id);
-
-    expect(mapInNewSession.core.isStreaming()).toBe(false);
+    const mapOnServer = jazzCloud.node.getCoValue(map.id);
+    expect(mapOnServer.isStreaming()).toBe(false);
+    expect(mapOnServer.knownState()).toEqual(
+      mapOnServer.knownStateWithStreaming(),
+    );
   });
 
   test("should be false when getting streaming content that's not really streaming", async () => {