cojson 0.16.3 → 0.16.5

package/src/coValueContentMessage.ts

@@ -0,0 +1,73 @@
+import {
+  CoValueHeader,
+  Transaction,
+  VerifiedState,
+} from "./coValueCore/verifiedState.js";
+import { MAX_RECOMMENDED_TX_SIZE } from "./config.js";
+import { Signature } from "./crypto/crypto.js";
+import { RawCoID, SessionID } from "./ids.js";
+import { getPriorityFromHeader } from "./priority.js";
+import { NewContentMessage, emptyKnownState } from "./sync.js";
+
+export function createContentMessage(
+  id: RawCoID,
+  header: CoValueHeader,
+  includeHeader = true,
+): NewContentMessage {
+  return {
+    action: "content",
+    id,
+    header: includeHeader ? header : undefined,
+    priority: getPriorityFromHeader(header),
+    new: {},
+  };
+}
+
+export function addTransactionToContentMessage(
+  content: NewContentMessage,
+  transaction: Transaction,
+  sessionID: SessionID,
+  signature: Signature,
+  txIdx: number,
+) {
+  const sessionContent = content.new[sessionID];
+
+  if (sessionContent) {
+    sessionContent.newTransactions.push(transaction);
+    sessionContent.lastSignature = signature;
+  } else {
+    content.new[sessionID] = {
+      after: txIdx,
+      newTransactions: [transaction],
+      lastSignature: signature,
+    };
+  }
+}
+
+export function getTransactionSize(transaction: Transaction) {
+  return transaction.privacy === "private"
+    ? transaction.encryptedChanges.length
+    : transaction.changes.length;
+}
+
+export function exceedsRecommendedSize(
+  baseSize: number,
+  transactionSize?: number,
+) {
+  if (transactionSize === undefined) {
+    return baseSize > MAX_RECOMMENDED_TX_SIZE;
+  }
+
+  return baseSize + transactionSize > MAX_RECOMMENDED_TX_SIZE;
+}
+
+export function knownStateFromContent(content: NewContentMessage) {
+  const knownState = emptyKnownState(content.id);
+
+  for (const [sessionID, session] of Object.entries(content.new)) {
+    knownState.sessions[sessionID as SessionID] =
+      session.after + session.newTransactions.length;
+  }
+
+  return knownState;
+}

package/src/coValueCore/coValueCore.ts

@@ -1,10 +1,10 @@
 import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
 import { Result, err } from "neverthrow";
-import { PeerState } from "../PeerState.js";
-import { RawCoValue } from "../coValue.js";
-import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
-import { RawGroup } from "../coValues/group.js";
-import { CO_VALUE_LOADING_CONFIG, MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import type { PeerState } from "../PeerState.js";
+import type { RawCoValue } from "../coValue.js";
+import type { ControlledAccountOrAgent } from "../coValues/account.js";
+import type { RawGroup } from "../coValues/group.js";
+import { CO_VALUE_LOADING_CONFIG } from "../config.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
@@ -16,25 +16,15 @@ import {
   SignerID,
   StreamingHash,
 } from "../crypto/crypto.js";
-import {
-  RawCoID,
-  SessionID,
-  TransactionID,
-  getParentGroupId,
-  isParentGroupReference,
-} from "../ids.js";
+import { RawCoID, SessionID, TransactionID } from "../ids.js";
 import { parseJSON, stableStringify } from "../jsonStringify.js";
 import { JsonValue } from "../jsonValue.js";
 import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
 import { logger } from "../logger.js";
-import {
-  determineValidTransactions,
-  isKeyForKeyField,
-} from "../permissions.js";
+import { determineValidTransactions } from "../permissions.js";
 import { CoValueKnownState, PeerID, emptyKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "../typeUtils/expectGroup.js";
-import { isAccountID } from "../typeUtils/isAccountID.js";
 import { getDependedOnCoValuesFromRawData } from "./utils.js";
 import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 
@@ -53,8 +43,6 @@ export type DecryptedTransaction = {
   trusting?: boolean;
 };
 
-const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
-
 export type AvailableCoValueCore = CoValueCore & { verified: VerifiedState };
 
 export class CoValueCore {
@@ -380,7 +368,7 @@ export class CoValueCore {
   }
 
   knownStateWithStreaming(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownStateWithStreaming();
     } else {
       return emptyKnownState(this.id);
@@ -388,7 +376,7 @@ export class CoValueCore {
   }
 
   knownState(): CoValueKnownState {
-    if (this.isAvailable()) {
+    if (this.verified) {
       return this.verified.knownState();
     } else {
       return emptyKnownState(this.id);
@@ -605,8 +593,17 @@ export class CoValueCore {
     )._unsafeUnwrap({ withStackTrace: true });
 
     if (success) {
+      const session = this.verified.sessions.get(sessionID);
+      const txIdx = session ? session.transactions.length - 1 : 0;
+
       this.node.syncManager.recordTransactionsSize([transaction], "local");
-      void this.node.syncManager.requestCoValueSync(this);
+      this.node.syncManager.syncLocalTransaction(
+        this.verified,
+        transaction,
+        sessionID,
+        signature,
+        txIdx,
+      );
     }
 
     return success;
@@ -759,20 +756,7 @@ export class CoValueCore {
     }
 
     if (this.verified.header.ruleset.type === "group") {
-      const content = expectGroup(this.getCurrentContent());
-
-      const currentKeyId = content.getCurrentReadKeyId();
-
-      if (!currentKeyId) {
-        throw new Error("No readKey set");
-      }
-
-      const secret = this.getReadKey(currentKeyId);
-
-      return {
-        secret: secret,
-        id: currentKeyId,
-      };
+      return expectGroup(this.getCurrentContent()).getCurrentReadKey();
     } else if (this.verified.header.ruleset.type === "ownedByGroup") {
       return this.node
         .expectCoValueLoaded(this.verified.header.ruleset.group)
@@ -784,154 +768,36 @@ export class CoValueCore {
     }
   }
 
+  readKeyCache = new Map<KeyID, KeySecret>();
   getReadKey(keyID: KeyID): KeySecret | undefined {
-    let key = readKeyCache.get(this)?.[keyID];
-    if (!key) {
-      key = this.getUncachedReadKey(keyID);
-      if (key) {
-        let cache = readKeyCache.get(this);
-        if (!cache) {
-          cache = {};
-          readKeyCache.set(this, cache);
-        }
-        cache[keyID] = key;
-      }
+    // We want to check the cache here, to skip re-computing the group content
+    const cachedSecret = this.readKeyCache.get(keyID);
+
+    if (cachedSecret) {
+      return cachedSecret;
     }
-    return key;
-  }
 
-  getUncachedReadKey(keyID: KeyID): KeySecret | undefined {
     if (!this.verified) {
       throw new Error(
         "CoValueCore: getUncachedReadKey called on coValue without verified state",
       );
     }
 
+    // Getting the readKey from accounts
     if (this.verified.header.ruleset.type === "group") {
       const content = expectGroup(
-        this.getCurrentContent({ ignorePrivateTransactions: true }), // to prevent recursion
-      );
-      const keyForEveryone = content.get(`${keyID}_for_everyone`);
-      if (keyForEveryone) {
-        return keyForEveryone;
-      }
-
-      // Try to find key revelation for us
-      const currentAgentOrAccountID = accountOrAgentIDfromSessionID(
-        this.node.currentSessionID,
+        // load the account without private transactions, because we are here
+        // to be able to decrypt those
+        this.getCurrentContent({ ignorePrivateTransactions: true }),
       );
 
-      // being careful here to avoid recursion
-      const lookupAccountOrAgentID = isAccountID(currentAgentOrAccountID)
-        ? this.id === currentAgentOrAccountID
-          ? this.crypto.getAgentID(this.node.agentSecret) // in accounts, the read key is revealed for the primitive agent
-          : currentAgentOrAccountID // current account ID
-        : currentAgentOrAccountID; // current agent ID
-
-      const lastReadyKeyEdit = content.lastEditAt(
-        `${keyID}_for_${lookupAccountOrAgentID}`,
-      );
-
-      if (lastReadyKeyEdit?.value) {
-        const revealer = lastReadyKeyEdit.by;
-        const revealerAgent = this.node
-          .resolveAccountAgent(revealer, "Expected to know revealer")
-          ._unsafeUnwrap({ withStackTrace: true });
-
-        const secret = this.crypto.unseal(
-          lastReadyKeyEdit.value,
-          this.crypto.getAgentSealerSecret(this.node.agentSecret), // being careful here to avoid recursion
-          this.crypto.getAgentSealerID(revealerAgent),
-          {
-            in: this.id,
-            tx: lastReadyKeyEdit.tx,
-          },
-        );
-
-        if (secret) {
-          return secret as KeySecret;
-        }
-      }
-
-      // Try to find indirect revelation through previousKeys
-
-      for (const co of content.keys()) {
-        if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-          const encryptingKeyID = co.split("_for_")[1] as KeyID;
-          const encryptingKeySecret = this.getReadKey(encryptingKeyID);
-
-          if (!encryptingKeySecret) {
-            continue;
-          }
-
-          const encryptedPreviousKey = content.get(co)!;
-
-          const secret = this.crypto.decryptKeySecret(
-            {
-              encryptedID: keyID,
-              encryptingID: encryptingKeyID,
-              encrypted: encryptedPreviousKey,
-            },
-            encryptingKeySecret,
-          );
-
-          if (secret) {
-            return secret as KeySecret;
-          } else {
-            logger.warn(
-              `Encrypting ${encryptingKeyID} key didn't decrypt ${keyID}`,
-            );
-          }
-        }
-      }
-
-      // try to find revelation to parent group read keys
-      for (const co of content.keys()) {
-        if (isParentGroupReference(co)) {
-          const parentGroupID = getParentGroupId(co);
-          const parentGroup = this.node.expectCoValueLoaded(
-            parentGroupID,
-            "Expected parent group to be loaded",
-          );
-
-          const parentKeys = this.findValidParentKeys(
-            keyID,
-            content,
-            parentGroup,
-          );
-
-          for (const parentKey of parentKeys) {
-            const revelationForParentKey = content.get(
-              `${keyID}_for_${parentKey.id}`,
-            );
-
-            if (revelationForParentKey) {
-              const secret = parentGroup.crypto.decryptKeySecret(
-                {
-                  encryptedID: keyID,
-                  encryptingID: parentKey.id,
-                  encrypted: revelationForParentKey,
-                },
-                parentKey.secret,
-              );
-
-              if (secret) {
-                return secret as KeySecret;
-              } else {
-                logger.warn(
-                  `Encrypting parent ${parentKey.id} key didn't decrypt ${keyID}`,
-                );
-              }
-            }
-          }
-        }
-      }
-
-      return undefined;
+      return content.getReadKey(keyID);
     } else if (this.verified.header.ruleset.type === "ownedByGroup") {
-      return this.node
-        .expectCoValueLoaded(this.verified.header.ruleset.group)
-        .getReadKey(keyID);
+      return expectGroup(
+        this.node
+          .expectCoValueLoaded(this.verified.header.ruleset.group)
+          .getCurrentContent(),
+      ).getReadKey(keyID);
     } else {
       throw new Error(
         "Only groups or values owned by groups have read secrets",
@@ -939,28 +805,6 @@ export class CoValueCore {
     }
   }
 
-  findValidParentKeys(keyID: KeyID, group: RawGroup, parentGroup: CoValueCore) {
-    const validParentKeys: { id: KeyID; secret: KeySecret }[] = [];
-
-    for (const co of group.keys()) {
-      if (isKeyForKeyField(co) && co.startsWith(keyID)) {
-        const encryptingKeyID = co.split("_for_")[1] as KeyID;
-        const encryptingKeySecret = parentGroup.getReadKey(encryptingKeyID);
-
-        if (!encryptingKeySecret) {
-          continue;
-        }
-
-        validParentKeys.push({
-          id: encryptingKeyID,
-          secret: encryptingKeySecret,
-        });
-      }
-    }
-
-    return validParentKeys;
-  }
-
   getGroup(): RawGroup {
     if (!this.verified) {
       throw new Error(

package/src/coValueCore/verifiedState.ts

@@ -1,6 +1,10 @@
 import { Result, err, ok } from "neverthrow";
 import { AnyRawCoValue } from "../coValue.js";
-import { MAX_RECOMMENDED_TX_SIZE } from "../config.js";
+import {
+  createContentMessage,
+  exceedsRecommendedSize,
+  getTransactionSize,
+} from "../coValueContentMessage.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -14,7 +18,6 @@ import { RawCoID, SessionID, TransactionID } from "../ids.js";
 import { Stringified } from "../jsonStringify.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { PermissionsDef as RulesetDef } from "../permissions.js";
-import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { InvalidHashError, InvalidSignatureError } from "./coValueCore.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
@@ -151,6 +154,17 @@ export class VerifiedState {
     return ok(true as const);
   }
 
+  getLastSignatureCheckpoint(sessionID: SessionID): number {
+    const sessionLog = this.sessions.get(sessionID);
+
+    if (!sessionLog?.signatureAfter) return -1;
+
+    return Object.keys(sessionLog.signatureAfter).reduce(
+      (max, idx) => Math.max(max, parseInt(idx)),
+      -1,
+    );
+  }
+
   private doAddTransactions(
     sessionID: SessionID,
     newTransactions: Transaction[],
@@ -165,24 +179,14 @@ export class VerifiedState {
     }
 
     const signatureAfter = sessionLog?.signatureAfter ?? {};
-
-    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
-      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
-      -1,
-    );
+    const lastInbetweenSignatureIdx =
+      this.getLastSignatureCheckpoint(sessionID);
 
     const sizeOfTxsSinceLastInbetweenSignature = transactions
       .slice(lastInbetweenSignatureIdx + 1)
-      .reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
+      .reduce((sum, tx) => sum + getTransactionSize(tx), 0);
 
-    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
+    if (exceedsRecommendedSize(sizeOfTxsSinceLastInbetweenSignature)) {
       signatureAfter[transactions.length - 1] = newSignature;
     }
 
@@ -242,13 +246,11 @@ export class VerifiedState {
       return this._cachedNewContentSinceEmpty;
     }
 
-    let currentPiece: NewContentMessage = {
-      action: "content",
-      id: this.id,
-      header: knownState?.header ? undefined : this.header,
-      priority: getPriorityFromHeader(this.header),
-      new: {},
-    };
+    let currentPiece: NewContentMessage = createContentMessage(
+      this.id,
+      this.header,
+      !knownState?.header,
+    );
 
     const pieces = [currentPiece];
 
@@ -299,25 +301,16 @@ export class VerifiedState {
         const oldPieceSize = pieceSize;
         for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
           const tx = log.transactions[txIdx]!;
-          pieceSize +=
-            tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length;
+          pieceSize += getTransactionSize(tx);
         }
 
-        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+        if (exceedsRecommendedSize(pieceSize)) {
           if (!currentPiece.expectContentUntil && pieces.length === 1) {
             currentPiece.expectContentUntil =
               this.knownStateWithStreaming().sessions;
           }
 
-          currentPiece = {
-            action: "content",
-            id: this.id,
-            header: undefined,
-            new: {},
-            priority: getPriorityFromHeader(this.header),
-          };
+          currentPiece = createContentMessage(this.id, this.header, false);
           pieces.push(currentPiece);
           pieceSize = pieceSize - oldPieceSize;
         }