cojson 0.8.32 → 0.8.35

package/src/permissions.ts

@@ -2,9 +2,16 @@ import { CoID } from "./coValue.js";
 import { CoValueCore, Transaction } from "./coValueCore.js";
 import { RawAccount, RawAccountID, RawProfile } from "./coValues/account.js";
 import { MapOpPayload } from "./coValues/coMap.js";
-import { EVERYONE, Everyone } from "./coValues/group.js";
+import { EVERYONE, Everyone, RawGroup } from "./coValues/group.js";
 import { KeyID } from "./crypto/crypto.js";
-import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
+import {
+  AgentID,
+  ParentGroupReference,
+  RawCoID,
+  SessionID,
+  TransactionID,
+  getParentGroupId,
+} from "./ids.js";
 import { parseJSON } from "./jsonStringify.js";
 import { JsonValue } from "./jsonValue.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
@@ -24,202 +31,20 @@ export type Role =
   | "writerInvite"
   | "readerInvite";
 
+type ValidTransactionsResult = { txID: TransactionID; tx: Transaction };
+type MemberState = { [agent: RawAccountID | AgentID]: Role; [EVERYONE]?: Role };
+
 export function determineValidTransactions(
   coValue: CoValueCore,
 ): { txID: TransactionID; tx: Transaction }[] {
   if (coValue.header.ruleset.type === "group") {
-    const allTransactionsSorted = [...coValue.sessionLogs.entries()].flatMap(
-      ([sessionID, sessionLog]) => {
-        return sessionLog.transactions.map((tx, txIndex) => ({
-          sessionID,
-          txIndex,
-          tx,
-        })) as {
-          sessionID: SessionID;
-          txIndex: number;
-          tx: Transaction;
-        }[];
-      },
-    );
-
-    allTransactionsSorted.sort((a, b) => {
-      return a.tx.madeAt - b.tx.madeAt;
-    });
-
     const initialAdmin = coValue.header.ruleset.initialAdmin;
-
     if (!initialAdmin) {
       throw new Error("Group must have initialAdmin");
     }
 
-    const memberState: {
-      [agent: RawAccountID | AgentID]: Role;
-      [EVERYONE]?: Role;
-    } = {};
-
-    const validTransactions: { txID: TransactionID; tx: Transaction }[] = [];
-
-    for (const { sessionID, txIndex, tx } of allTransactionsSorted) {
-      // console.log("before", { memberState, validTransactions });
-      const transactor = accountOrAgentIDfromSessionID(sessionID);
-
-      if (tx.privacy === "private") {
-        if (memberState[transactor] === "admin") {
-          validTransactions.push({
-            txID: { sessionID, txIndex },
-            tx,
-          });
-          continue;
-        } else {
-          console.warn("Only admins can make private transactions in groups");
-          continue;
-        }
-      }
-
-      let changes;
-
-      try {
-        changes = parseJSON(tx.changes);
-      } catch (e) {
-        console.warn(
-          coValue.id,
-          "Invalid JSON in transaction",
-          e,
-          tx,
-          JSON.stringify(tx.changes, (k, v) =>
-            k === "changes" || k === "encryptedChanges"
-              ? v.slice(0, 20) + "..."
-              : v,
-          ),
-        );
-        continue;
-      }
-
-      const change = changes[0] as
-        | MapOpPayload<RawAccountID | AgentID | Everyone, Role>
-        | MapOpPayload<"readKey", JsonValue>
-        | MapOpPayload<"profile", CoID<RawProfile>>;
-      if (changes.length !== 1) {
-        console.warn("Group transaction must have exactly one change");
-        continue;
-      }
-
-      if (change.op !== "set") {
-        console.warn("Group transaction must set a role or readKey");
-        continue;
-      }
-
-      if (change.key === "readKey") {
-        if (memberState[transactor] !== "admin") {
-          console.warn("Only admins can set readKeys");
-          continue;
-        }
-
-        validTransactions.push({ txID: { sessionID, txIndex }, tx });
-        continue;
-      } else if (change.key === "profile") {
-        if (memberState[transactor] !== "admin") {
-          console.warn("Only admins can set profile");
-          continue;
-        }
-
-        validTransactions.push({ txID: { sessionID, txIndex }, tx });
-        continue;
-      } else if (
-        isKeyForKeyField(change.key) ||
-        isKeyForAccountField(change.key)
-      ) {
-        if (
-          memberState[transactor] !== "admin" &&
-          memberState[transactor] !== "adminInvite" &&
-          memberState[transactor] !== "writerInvite" &&
-          memberState[transactor] !== "readerInvite"
-        ) {
-          console.warn("Only admins can reveal keys");
-          continue;
-        }
-
-        // TODO: check validity of agents who the key is revealed to?
-
-        validTransactions.push({ txID: { sessionID, txIndex }, tx });
-        continue;
-      }
-
-      const affectedMember = change.key;
-      const assignedRole = change.value;
-
-      if (
-        change.value !== "admin" &&
-        change.value !== "writer" &&
-        change.value !== "reader" &&
-        change.value !== "revoked" &&
-        change.value !== "adminInvite" &&
-        change.value !== "writerInvite" &&
-        change.value !== "readerInvite"
-      ) {
-        console.warn("Group transaction must set a valid role");
-        continue;
-      }
-
-      if (
-        affectedMember === EVERYONE &&
-        !(
-          change.value === "reader" ||
-          change.value === "writer" ||
-          change.value === "revoked"
-        )
-      ) {
-        console.warn("Everyone can only be set to reader, writer or revoked");
-        continue;
-      }
-
-      const isFirstSelfAppointment =
-        !memberState[transactor] &&
-        transactor === initialAdmin &&
-        change.op === "set" &&
-        change.key === transactor &&
-        change.value === "admin";
-
-      if (!isFirstSelfAppointment) {
-        if (memberState[transactor] === "admin") {
-          if (
-            memberState[affectedMember] === "admin" &&
-            affectedMember !== transactor &&
-            assignedRole !== "admin"
-          ) {
-            console.warn("Admins can only demote themselves.");
-            continue;
-          }
-        } else if (memberState[transactor] === "adminInvite") {
-          if (change.value !== "admin") {
-            console.warn("AdminInvites can only create admins.");
-            continue;
-          }
-        } else if (memberState[transactor] === "writerInvite") {
-          if (change.value !== "writer") {
-            console.warn("WriterInvites can only create writers.");
-            continue;
-          }
-        } else if (memberState[transactor] === "readerInvite") {
-          if (change.value !== "reader") {
-            console.warn("ReaderInvites can only create reader.");
-            continue;
-          }
-        } else {
-          console.warn(
-            "Group transaction must be made by current admin or invite",
-          );
-          continue;
-        }
-      }
-
-      memberState[affectedMember] = change.value;
-      validTransactions.push({ txID: { sessionID, txIndex }, tx });
-
-      // console.log("after", { memberState, validTransactions });
-    }
-
-    return validTransactions;
+    return determineValidTransactionsForGroup(coValue, initialAdmin)
+      .validTransactions;
   } else if (coValue.header.ruleset.type === "ownedByGroup") {
     const groupContent = expectGroup(
       coValue.node
@@ -241,27 +66,18 @@ export function determineValidTransactions(
         return sessionLog.transactions
           .filter((tx) => {
             const groupAtTime = groupContent.atTime(tx.madeAt);
-            const effectiveTransactor =
-              transactor === groupContent.id &&
-              groupAtTime instanceof RawAccount
-                ? groupAtTime.currentAgentID().match(
-                    (agentID) => agentID,
-                    (e) => {
-                      console.error(
-                        "Error while determining current agent ID in valid transactions",
-                        e,
-                      );
-                      return undefined;
-                    },
-                  )
-                : transactor;
+            const effectiveTransactor = agentInAccountOrMemberInGroup(
+              transactor,
+              groupAtTime,
+            );
 
             if (!effectiveTransactor) {
               return false;
             }
 
             const transactorRoleAtTxTime =
-              groupAtTime.get(effectiveTransactor) || groupAtTime.get(EVERYONE);
+              groupAtTime.roleOfInternal(effectiveTransactor)?.role ||
+              groupAtTime.roleOfInternal(EVERYONE)?.role;
 
             return (
               transactorRoleAtTxTime === "admin" ||
@@ -291,6 +107,275 @@ export function determineValidTransactions(
   }
 }
 
+function isHigherRole(a: Role, b: Role | undefined) {
+  if (a === undefined) return false;
+  if (b === undefined) return true;
+  if (b === "admin") return false;
+  if (a === "admin") return true;
+
+  return a === "writer" && b === "reader";
+}
+
+function resolveMemberStateFromParentReference(
+  coValue: CoValueCore,
+  memberState: MemberState,
+  parentReference: ParentGroupReference,
+) {
+  const parentGroup = coValue.node.expectCoValueLoaded(
+    getParentGroupId(parentReference),
+    "Expected parent group to be loaded",
+  );
+
+  if (parentGroup.header.ruleset.type !== "group") {
+    return;
+  }
+
+  const initialAdmin = parentGroup.header.ruleset.initialAdmin;
+
+  if (!initialAdmin) {
+    throw new Error("Group must have initialAdmin");
+  }
+
+  const { memberState: parentGroupMemberState } =
+    determineValidTransactionsForGroup(parentGroup, initialAdmin);
+
+  for (const agent of Object.keys(parentGroupMemberState) as Array<
+    keyof MemberState
+  >) {
+    const parentRole = parentGroupMemberState[agent];
+    const currentRole = memberState[agent];
+
+    if (parentRole && isHigherRole(parentRole, currentRole)) {
+      memberState[agent] = parentRole;
+    }
+  }
+}
+
+function determineValidTransactionsForGroup(
+  coValue: CoValueCore,
+  initialAdmin: RawAccountID | AgentID,
+): { validTransactions: ValidTransactionsResult[]; memberState: MemberState } {
+  const allTransactionsSorted = [...coValue.sessionLogs.entries()].flatMap(
+    ([sessionID, sessionLog]) => {
+      return sessionLog.transactions.map((tx, txIndex) => ({
+        sessionID,
+        txIndex,
+        tx,
+      })) as {
+        sessionID: SessionID;
+        txIndex: number;
+        tx: Transaction;
+      }[];
+    },
+  );
+
+  allTransactionsSorted.sort((a, b) => {
+    return a.tx.madeAt - b.tx.madeAt;
+  });
+
+  const memberState: MemberState = {};
+  const validTransactions: ValidTransactionsResult[] = [];
+
+  for (const { sessionID, txIndex, tx } of allTransactionsSorted) {
+    // console.log("before", { memberState, validTransactions });
+    const transactor = accountOrAgentIDfromSessionID(sessionID);
+
+    if (tx.privacy === "private") {
+      if (memberState[transactor] === "admin") {
+        validTransactions.push({
+          txID: { sessionID, txIndex },
+          tx,
+        });
+        continue;
+      } else {
+        console.warn("Only admins can make private transactions in groups");
+        continue;
+      }
+    }
+
+    let changes;
+
+    try {
+      changes = parseJSON(tx.changes);
+    } catch (e) {
+      console.warn(
+        coValue.id,
+        "Invalid JSON in transaction",
+        e,
+        tx,
+        JSON.stringify(tx.changes, (k, v) =>
+          k === "changes" || k === "encryptedChanges"
+            ? v.slice(0, 20) + "..."
+            : v,
+        ),
+      );
+      continue;
+    }
+
+    const change = changes[0] as
+      | MapOpPayload<RawAccountID | AgentID | Everyone, Role>
+      | MapOpPayload<"readKey", JsonValue>
+      | MapOpPayload<"profile", CoID<RawProfile>>
+      | MapOpPayload<`parent_${CoID<RawGroup>}`, CoID<RawGroup>>
+      | MapOpPayload<`child_${CoID<RawGroup>}`, CoID<RawGroup>>;
+
+    if (changes.length !== 1) {
+      console.warn("Group transaction must have exactly one change");
+      continue;
+    }
+
+    if (change.op !== "set") {
+      console.warn("Group transaction must set a role or readKey");
+      continue;
+    }
+
+    if (change.key === "readKey") {
+      if (memberState[transactor] !== "admin") {
+        console.warn("Only admins can set readKeys");
+        continue;
+      }
+
+      validTransactions.push({ txID: { sessionID, txIndex }, tx });
+      continue;
+    } else if (change.key === "profile") {
+      if (memberState[transactor] !== "admin") {
+        console.warn("Only admins can set profile");
+        continue;
+      }
+
+      validTransactions.push({ txID: { sessionID, txIndex }, tx });
+      continue;
+    } else if (
+      isKeyForKeyField(change.key) ||
+      isKeyForAccountField(change.key)
+    ) {
+      if (
+        memberState[transactor] !== "admin" &&
+        memberState[transactor] !== "adminInvite" &&
+        memberState[transactor] !== "writerInvite" &&
+        memberState[transactor] !== "readerInvite"
+      ) {
+        console.warn("Only admins can reveal keys");
+        continue;
+      }
+
+      // TODO: check validity of agents who the key is revealed to?
+
+      validTransactions.push({ txID: { sessionID, txIndex }, tx });
+      continue;
+    } else if (isParentExtension(change.key)) {
+      if (memberState[transactor] !== "admin") {
+        console.warn("Only admins can set parent extensions");
+        continue;
+      }
+      resolveMemberStateFromParentReference(coValue, memberState, change.key);
+      validTransactions.push({ txID: { sessionID, txIndex }, tx });
+      continue;
+    } else if (isChildExtension(change.key)) {
+      if (memberState[transactor] !== "admin") {
+        console.warn("Only admins can set child extensions");
+        continue;
+      }
+      validTransactions.push({ txID: { sessionID, txIndex }, tx });
+      continue;
+    }
+
+    const affectedMember = change.key;
+    const assignedRole = change.value;
+
+    if (
+      change.value !== "admin" &&
+      change.value !== "writer" &&
+      change.value !== "reader" &&
+      change.value !== "revoked" &&
+      change.value !== "adminInvite" &&
+      change.value !== "writerInvite" &&
+      change.value !== "readerInvite"
+    ) {
+      console.warn("Group transaction must set a valid role");
+      continue;
+    }
+
+    if (
+      affectedMember === EVERYONE &&
+      !(
+        change.value === "reader" ||
+        change.value === "writer" ||
+        change.value === "revoked"
+      )
+    ) {
+      console.warn("Everyone can only be set to reader, writer or revoked");
+      continue;
+    }
+
+    const isFirstSelfAppointment =
+      !memberState[transactor] &&
+      transactor === initialAdmin &&
+      change.op === "set" &&
+      change.key === transactor &&
+      change.value === "admin";
+
+    if (!isFirstSelfAppointment) {
+      if (memberState[transactor] === "admin") {
+        if (
+          memberState[affectedMember] === "admin" &&
+          affectedMember !== transactor &&
+          assignedRole !== "admin"
+        ) {
+          console.warn("Admins can only demote themselves.");
+          continue;
+        }
+      } else if (memberState[transactor] === "adminInvite") {
+        if (change.value !== "admin") {
+          console.warn("AdminInvites can only create admins.");
+          continue;
+        }
+      } else if (memberState[transactor] === "writerInvite") {
+        if (change.value !== "writer") {
+          console.warn("WriterInvites can only create writers.");
+          continue;
+        }
+      } else if (memberState[transactor] === "readerInvite") {
+        if (change.value !== "reader") {
+          console.warn("ReaderInvites can only create reader.");
+          continue;
+        }
+      } else {
+        console.warn(
+          "Group transaction must be made by current admin or invite",
+        );
+        continue;
+      }
+    }
+
+    memberState[affectedMember] = change.value;
+    validTransactions.push({ txID: { sessionID, txIndex }, tx });
+
+    // console.log("after", { memberState, validTransactions });
+  }
+
+  return { validTransactions, memberState };
+}
+
+function agentInAccountOrMemberInGroup(
+  transactor: RawAccountID | AgentID,
+  groupAtTime: RawGroup,
+): RawAccountID | AgentID | undefined {
+  if (transactor === groupAtTime.id && groupAtTime instanceof RawAccount) {
+    return groupAtTime.currentAgentID().match(
+      (agentID) => agentID,
+      (e) => {
+        console.error(
+          "Error while determining current agent ID in valid transactions",
+          e,
+        );
+        return undefined;
+      },
+    );
+  }
+  return transactor;
+}
+
 export function isKeyForKeyField(co: string): co is `${KeyID}_for_${KeyID}` {
   return co.startsWith("key_") && co.includes("_for_key");
 }
@@ -304,3 +389,11 @@ export function isKeyForAccountField(
     co.includes("_for_everyone")
   );
 }
+
+function isParentExtension(key: string): key is `parent_${CoID<RawGroup>}` {
+  return key.startsWith("parent_");
+}
+
+function isChildExtension(key: string): key is `child_${CoID<RawGroup>}` {
+  return key.startsWith("child_");
+}

package/src/storage/index.ts

@@ -146,7 +146,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         asDependencyOf || id,
       );
     } else if (!known?.header && coValue.header?.ruleset.type === "group") {
-      const dependedOnAccounts = new Set();
+      const dependedOnAccountsAndGroups = new Set();
       for (const session of Object.values(coValue.sessionEntries)) {
         for (const entry of session) {
           for (const tx of entry.transactions) {
@@ -154,16 +154,24 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
               const parsedChanges = JSON.parse(tx.changes);
               for (const change of parsedChanges) {
                 if (change.op === "set" && change.key.startsWith("co_")) {
-                  dependedOnAccounts.add(change.key);
+                  dependedOnAccountsAndGroups.add(change.key);
+                }
+                if (
+                  change.op === "set" &&
+                  change.key.startsWith("parent_co_")
+                ) {
+                  dependedOnAccountsAndGroups.add(
+                    change.key.replace("parent_", ""),
+                  );
                 }
               }
             }
           }
         }
       }
-      for (const account of dependedOnAccounts) {
+      for (const accountOrGroup of dependedOnAccountsAndGroups) {
         await this.sendNewContent(
-          account as CoID<RawCoValue>,
+          accountOrGroup as CoID<RawCoValue>,
           undefined,
           asDependencyOf || id,
         );

package/src/sync.ts

@@ -304,14 +304,19 @@ export class SyncManager {
 
     if (peerState.isServerOrStoragePeer()) {
       const initialSync = async () => {
-        for (const id of this.local.coValuesStore.getKeys()) {
-          // console.log("subscribing to after peer added", id, peer.id)
-          await this.subscribeToIncludingDependencies(id, peerState);
+        for (const entry of this.local.coValuesStore.getValues()) {
+          await this.subscribeToIncludingDependencies(entry.id, peerState);
 
-          peerState.optimisticKnownStates.dispatch({
-            type: "SET_AS_EMPTY",
-            id,
-          });
+          if (entry.state.type === "available") {
+            await this.sendNewContentIncludingDependencies(entry.id, peerState);
+          }
+
+          if (!peerState.optimisticKnownStates.has(entry.id)) {
+            peerState.optimisticKnownStates.dispatch({
+              type: "SET_AS_EMPTY",
+              id: entry.id,
+            });
+          }
         }
       };
       void initialSync();
@@ -403,27 +408,39 @@ export class SyncManager {
     }
 
     if (entry.state.type === "loading") {
-      const value = await entry.getCoValue();
+      // We need to return from handleLoad immediately and wait for the CoValue to be loaded
+      // in a new task, otherwise we might block further incoming content messages that would
+      // resolve the CoValue as available. This can happen when we receive fresh
+      // content from a client, but we are a server with our own upstream server(s)
+      entry
+        .getCoValue()
+        .then(async (value) => {
+          if (value === "unavailable") {
+            peer.dispatchToKnownStates({
+              type: "SET",
+              id: msg.id,
+              value: knownStateIn(msg),
+            });
+            peer.toldKnownState.add(msg.id);
+
+            this.trySendToPeer(peer, {
+              action: "known",
+              id: msg.id,
+              header: false,
+              sessions: {},
+            }).catch((e) => {
+              console.error("Error sending known state back", e);
+            });
 
-      if (value === "unavailable") {
-        peer.dispatchToKnownStates({
-          type: "SET",
-          id: msg.id,
-          value: knownStateIn(msg),
-        });
-        peer.toldKnownState.add(msg.id);
-
-        this.trySendToPeer(peer, {
-          action: "known",
-          id: msg.id,
-          header: false,
-          sessions: {},
-        }).catch((e) => {
-          console.error("Error sending known state back", e);
-        });
+            return;
+          }
 
-        return;
-      }
+          await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
+          await this.sendNewContentIncludingDependencies(msg.id, peer);
+        })
+        .catch((e) => {
+          console.error("Error loading coValue in handleLoad loading state", e);
+        });
     }
 
     if (entry.state.type === "available") {
@@ -462,15 +479,7 @@ export class SyncManager {
                 e,
               );
             });
-        } else {
-          throw new Error(
-            "Expected coValue dependency entry to be created, missing subscribe?",
-          );
         }
-      } else {
-        throw new Error(
-          `Expected coValue entry for ${msg.id} to be created on known state, missing subscribe?`,
-        );
       }
     }