cojson 0.13.16 → 0.13.18

package/src/coValueCore/verifiedState.ts

@@ -0,0 +1,376 @@
+import { Result, err, ok } from "neverthrow";
+import { AnyRawCoValue } from "../coValue.js";
+import {
+  CryptoProvider,
+  Encrypted,
+  Hash,
+  KeyID,
+  Signature,
+  SignerID,
+  StreamingHash,
+} from "../crypto/crypto.js";
+import { RawCoID, SessionID, TransactionID } from "../ids.js";
+import { Stringified } from "../jsonStringify.js";
+import { JsonObject, JsonValue } from "../jsonValue.js";
+import { PermissionsDef as RulesetDef } from "../permissions.js";
+import { getPriorityFromHeader } from "../priority.js";
+import { CoValueKnownState, NewContentMessage } from "../sync.js";
+import {
+  InvalidHashError,
+  InvalidSignatureError,
+  MAX_RECOMMENDED_TX_SIZE,
+} from "./coValueCore.js";
+import { TryAddTransactionsError } from "./coValueCore.js";
+
+export type CoValueHeader = {
+  type: AnyRawCoValue["type"];
+  ruleset: RulesetDef;
+  meta: JsonObject | null;
+} & CoValueUniqueness;
+
+export type CoValueUniqueness = {
+  uniqueness: JsonValue;
+  createdAt?: `2${string}` | null;
+};
+
+export type PrivateTransaction = {
+  privacy: "private";
+  madeAt: number;
+  keyUsed: KeyID;
+  encryptedChanges: Encrypted<JsonValue[], { in: RawCoID; tx: TransactionID }>;
+};
+
+export type TrustingTransaction = {
+  privacy: "trusting";
+  madeAt: number;
+  changes: Stringified<JsonValue[]>;
+};
+
+export type Transaction = PrivateTransaction | TrustingTransaction;
+
+type SessionLog = {
+  readonly transactions: Transaction[];
+  lastHash?: Hash;
+  streamingHash: StreamingHash;
+  readonly signatureAfter: { [txIdx: number]: Signature | undefined };
+  lastSignature: Signature;
+};
+
+export type ValidatedSessions = Map<SessionID, SessionLog>;
+
+export class VerifiedState {
+  readonly id: RawCoID;
+  readonly crypto: CryptoProvider;
+  readonly header: CoValueHeader;
+  readonly sessions: ValidatedSessions;
+  private _cachedKnownState?: CoValueKnownState;
+  private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
+
+  constructor(
+    id: RawCoID,
+    crypto: CryptoProvider,
+    header: CoValueHeader,
+    sessions: ValidatedSessions,
+  ) {
+    this.id = id;
+    this.crypto = crypto;
+    this.header = header;
+    this.sessions = sessions;
+  }
+
+  clone(): VerifiedState {
+    // do a deep clone, including the sessions
+    const clonedSessions = new Map();
+    for (let [sessionID, sessionLog] of this.sessions) {
+      clonedSessions.set(sessionID, {
+        lastSignature: sessionLog.lastSignature,
+        lastHash: sessionLog.lastHash,
+        streamingHash: sessionLog.streamingHash.clone(),
+        signatureAfter: { ...sessionLog.signatureAfter },
+        transactions: sessionLog.transactions.slice(),
+      } satisfies SessionLog);
+    }
+    return new VerifiedState(this.id, this.crypto, this.header, clonedSessions);
+  }
+
+  tryAddTransactions(
+    sessionID: SessionID,
+    signerID: SignerID,
+    newTransactions: Transaction[],
+    givenExpectedNewHash: Hash | undefined,
+    newSignature: Signature,
+    skipVerify: boolean = false,
+    givenNewStreamingHash?: StreamingHash,
+  ): Result<true, TryAddTransactionsError> {
+    if (skipVerify === true && givenNewStreamingHash && givenExpectedNewHash) {
+      this.doAddTransactions(
+        sessionID,
+        newTransactions,
+        newSignature,
+        givenExpectedNewHash,
+        givenNewStreamingHash,
+      );
+    } else {
+      const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
+        sessionID,
+        newTransactions,
+      );
+
+      if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
+        return err({
+          type: "InvalidHash",
+          id: this.id,
+          expectedNewHash,
+          givenExpectedNewHash,
+        } satisfies InvalidHashError);
+      }
+
+      if (!this.crypto.verify(newSignature, expectedNewHash, signerID)) {
+        return err({
+          type: "InvalidSignature",
+          id: this.id,
+          newSignature,
+          sessionID,
+          signerID,
+        } satisfies InvalidSignatureError);
+      }
+
+      this.doAddTransactions(
+        sessionID,
+        newTransactions,
+        newSignature,
+        expectedNewHash,
+        newStreamingHash,
+      );
+    }
+
+    return ok(true as const);
+  }
+
+  private doAddTransactions(
+    sessionID: SessionID,
+    newTransactions: Transaction[],
+    newSignature: Signature,
+    expectedNewHash: Hash,
+    newStreamingHash: StreamingHash,
+  ) {
+    const transactions = this.sessions.get(sessionID)?.transactions ?? [];
+
+    for (const tx of newTransactions) {
+      transactions.push(tx);
+    }
+
+    const signatureAfter = this.sessions.get(sessionID)?.signatureAfter ?? {};
+
+    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
+      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
+      -1,
+    );
+
+    const sizeOfTxsSinceLastInbetweenSignature = transactions
+      .slice(lastInbetweenSignatureIdx + 1)
+      .reduce(
+        (sum, tx) =>
+          sum +
+          (tx.privacy === "private"
+            ? tx.encryptedChanges.length
+            : tx.changes.length),
+        0,
+      );
+
+    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
+      signatureAfter[transactions.length - 1] = newSignature;
+    }
+
+    this.sessions.set(sessionID, {
+      transactions,
+      lastHash: expectedNewHash,
+      streamingHash: newStreamingHash,
+      lastSignature: newSignature,
+      signatureAfter: signatureAfter,
+    });
+
+    this._cachedNewContentSinceEmpty = undefined;
+    this._cachedKnownState = undefined;
+  }
+
+  expectedNewHashAfter(
+    sessionID: SessionID,
+    newTransactions: Transaction[],
+  ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
+    const streamingHash =
+      this.sessions.get(sessionID)?.streamingHash.clone() ??
+      new StreamingHash(this.crypto);
+
+    for (const transaction of newTransactions) {
+      streamingHash.update(transaction);
+    }
+
+    return {
+      expectedNewHash: streamingHash.digest(),
+      newStreamingHash: streamingHash,
+    };
+  }
+
+  newContentSince(
+    knownState: CoValueKnownState | undefined,
+  ): NewContentMessage[] | undefined {
+    const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
+
+    if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
+      return this._cachedNewContentSinceEmpty;
+    }
+
+    let currentPiece: NewContentMessage = {
+      action: "content",
+      id: this.id,
+      header: knownState?.header ? undefined : this.header,
+      priority: getPriorityFromHeader(this.header),
+      new: {},
+    };
+
+    const pieces = [currentPiece];
+
+    const sentState: CoValueKnownState["sessions"] = {};
+
+    let pieceSize = 0;
+
+    let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
+
+    while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
+      if (sessionsTodoAgain === "first") {
+        sessionsTodoAgain = undefined;
+      }
+      const sessionsTodo = sessionsTodoAgain ?? this.sessions.keys();
+
+      for (const sessionIDKey of sessionsTodo) {
+        const sessionID = sessionIDKey as SessionID;
+        const log = this.sessions.get(sessionID)!;
+        const knownStateForSessionID = knownState?.sessions[sessionID];
+        const sentStateForSessionID = sentState[sessionID];
+        const nextKnownSignatureIdx = getNextKnownSignatureIdx(
+          log,
+          knownStateForSessionID,
+          sentStateForSessionID,
+        );
+
+        const firstNewTxIdx =
+          sentStateForSessionID ?? knownStateForSessionID ?? 0;
+        const afterLastNewTxIdx =
+          nextKnownSignatureIdx === undefined
+            ? log.transactions.length
+            : nextKnownSignatureIdx + 1;
+
+        const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
+
+        if (nNewTx === 0) {
+          sessionsTodoAgain?.delete(sessionID);
+          continue;
+        }
+
+        if (afterLastNewTxIdx < log.transactions.length) {
+          if (!sessionsTodoAgain) {
+            sessionsTodoAgain = new Set();
+          }
+          sessionsTodoAgain.add(sessionID);
+        }
+
+        const oldPieceSize = pieceSize;
+        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
+          const tx = log.transactions[txIdx]!;
+          pieceSize +=
+            tx.privacy === "private"
+              ? tx.encryptedChanges.length
+              : tx.changes.length;
+        }
+
+        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+          currentPiece = {
+            action: "content",
+            id: this.id,
+            header: undefined,
+            new: {},
+            priority: getPriorityFromHeader(this.header),
+          };
+          pieces.push(currentPiece);
+          pieceSize = pieceSize - oldPieceSize;
+        }
+
+        let sessionEntry = currentPiece.new[sessionID];
+        if (!sessionEntry) {
+          sessionEntry = {
+            after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
+            newTransactions: [],
+            lastSignature: "WILL_BE_REPLACED" as Signature,
+          };
+          currentPiece.new[sessionID] = sessionEntry;
+        }
+
+        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
+          const tx = log.transactions[txIdx]!;
+          sessionEntry.newTransactions.push(tx);
+        }
+
+        sessionEntry.lastSignature =
+          nextKnownSignatureIdx === undefined
+            ? log.lastSignature!
+            : log.signatureAfter[nextKnownSignatureIdx]!;
+
+        sentState[sessionID] =
+          (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
+      }
+    }
+
+    const piecesWithContent = pieces.filter(
+      (piece) => Object.keys(piece.new).length > 0 || piece.header,
+    );
+
+    if (piecesWithContent.length === 0) {
+      return undefined;
+    }
+
+    if (isKnownStateEmpty) {
+      this._cachedNewContentSinceEmpty = piecesWithContent;
+    }
+
+    return piecesWithContent;
+  }
+
+  knownState(): CoValueKnownState {
+    if (this._cachedKnownState) {
+      return this._cachedKnownState;
+    } else {
+      const knownState = this.knownStateUncached();
+      this._cachedKnownState = knownState;
+      return knownState;
+    }
+  }
+
+  /** @internal */
+  knownStateUncached(): CoValueKnownState {
+    const sessions: CoValueKnownState["sessions"] = {};
+
+    for (const [sessionID, sessionLog] of this.sessions.entries()) {
+      sessions[sessionID] = sessionLog.transactions.length;
+    }
+
+    return {
+      id: this.id,
+      header: true,
+      sessions,
+    };
+  }
+}
+
+function getNextKnownSignatureIdx(
+  log: SessionLog,
+  knownStateForSessionID?: number,
+  sentStateForSessionID?: number,
+) {
+  return Object.keys(log.signatureAfter)
+    .map(Number)
+    .sort((a, b) => a - b)
+    .find(
+      (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
+    );
+}

package/src/coValues/account.ts

@@ -1,9 +1,12 @@
 import { CoID, RawCoValue } from "../coValue.js";
 import {
+  AvailableCoValueCore,
   CoValueCore,
+} from "../coValueCore/coValueCore.js";
+import {
   CoValueHeader,
   CoValueUniqueness,
-} from "../coValueCore.js";
+} from "../coValueCore/verifiedState.js";
 import {
   AgentSecret,
   CryptoProvider,
@@ -85,35 +88,20 @@ export interface ControlledAccountOrAgent {
 }
 
 /** @hidden */
-export class RawControlledAccount<Meta extends AccountMeta = AccountMeta>
-  extends RawAccount<Meta>
-  implements ControlledAccountOrAgent
-{
+export class ControlledAccount implements ControlledAccountOrAgent {
+  account: RawAccount<AccountMeta>;
   agentSecret: AgentSecret;
+  _cachedCurrentAgentID: AgentID | undefined;
   crypto: CryptoProvider;
 
-  constructor(core: CoValueCore, agentSecret: AgentSecret) {
-    super(core);
-
+  constructor(account: RawAccount<AccountMeta>, agentSecret: AgentSecret) {
+    this.account = account;
     this.agentSecret = agentSecret;
-    this.crypto = core.node.crypto;
-  }
-
-  /**
-   * Creates a new group (with the current account as the group's first admin).
-   * @category 1. High-level
-   */
-  createGroup(
-    uniqueness: CoValueUniqueness = this.core.crypto.createdNowUnique(),
-  ) {
-    return this.core.node.createGroup(uniqueness);
+    this.crypto = account.core.node.crypto;
   }
 
-  async acceptInvite<T extends RawCoValue>(
-    groupOrOwnedValueID: CoID<T>,
-    inviteSecret: InviteSecret,
-  ): Promise<void> {
-    return this.core.node.acceptInvite(groupOrOwnedValueID, inviteSecret);
+  get id(): RawAccountID {
+    return this.account.id;
   }
 
   currentAgentID(): AgentID {
@@ -186,7 +174,14 @@ export class RawProfile<
 > extends RawCoMap<Shape, Meta> {}
 
 export type RawAccountMigration<Meta extends AccountMeta = AccountMeta> = (
-  account: RawControlledAccount<Meta>,
+  account: RawAccount<Meta>,
   localNode: LocalNode,
   creationProps?: { name: string },
 ) => void | Promise<void>;
+
+export function expectAccount(content: RawCoValue): RawAccount {
+  if (!(content instanceof RawAccount)) {
+    throw new Error("Expected an account");
+  }
+  return content;
+}

package/src/coValues/coList.ts

@@ -1,7 +1,11 @@
 import { CoID, RawCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
+import {
+  AvailableCoValueCore,
+  CoValueCore,
+} from "../coValueCore/coValueCore.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
+import { CoValueKnownState } from "../sync.js";
 import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { isCoValue } from "../typeUtils/isCoValue.js";
 import { RawAccountID } from "./account.js";
@@ -41,7 +45,7 @@ type DeletionEntry = {
   deletionID: OpID;
 } & DeletionOpPayload;
 
-export class RawCoListView<
+export class RawCoList<
   Item extends JsonValue = JsonValue,
   Meta extends JsonObject | null = null,
 > implements RawCoValue
@@ -51,7 +55,7 @@ export class RawCoListView<
   /** @category 6. Meta */
   type: "colist" | "coplaintext" = "colist" as const;
   /** @category 6. Meta */
-  core: CoValueCore;
+  core: AvailableCoValueCore;
   /** @internal */
   afterStart: OpID[];
   /** @internal */
@@ -81,26 +85,52 @@ export class RawCoListView<
     madeAt: number;
     opID: OpID;
   }[];
+  /** @internal */
+  totalValidTransactions = 0;
+  knownTransactions: CoValueKnownState["sessions"] = {};
+  lastValidTransaction: number | undefined;
 
   /** @internal */
-  constructor(core: CoValueCore) {
+  constructor(core: AvailableCoValueCore) {
     this.id = core.id as CoID<this>;
     this.core = core;
-    this.afterStart = [];
-    this.beforeEnd = [];
-    this.insertions = {};
-    this.deletionsByInsertion = {};
 
     this.insertions = {};
     this.deletionsByInsertion = {};
     this.afterStart = [];
     this.beforeEnd = [];
+    this.knownTransactions = {};
+
+    this.processNewTransactions();
+  }
+
+  processNewTransactions() {
+    const transactions = this.core.getValidSortedTransactions({
+      ignorePrivateTransactions: false,
+      knownTransactions: this.knownTransactions,
+    });
+
+    if (transactions.length === 0) {
+      return;
+    }
+
+    this.totalValidTransactions += transactions.length;
+    let lastValidTransaction: number | undefined = undefined;
+    let oldestValidTransaction: number | undefined = undefined;
+    this._cachedEntries = undefined;
+
+    for (const { txID, changes, madeAt } of transactions) {
+      lastValidTransaction = Math.max(lastValidTransaction ?? 0, madeAt);
+      oldestValidTransaction = Math.min(
+        oldestValidTransaction ?? Infinity,
+        madeAt,
+      );
+
+      this.knownTransactions[txID.sessionID] = Math.max(
+        this.knownTransactions[txID.sessionID] ?? 0,
+        txID.txIndex,
+      );
 
-    for (const {
-      txID,
-      changes,
-      madeAt,
-    } of this.core.getValidSortedTransactions()) {
       for (const [changeIdx, changeUntyped] of changes.entries()) {
         const change = changeUntyped as ListOpPayload<Item>;
 
@@ -193,11 +223,21 @@ export class RawCoListView<
         }
       }
     }
+
+    if (
+      this.lastValidTransaction &&
+      oldestValidTransaction &&
+      oldestValidTransaction < this.lastValidTransaction
+    ) {
+      this.rebuildFromCore();
+    } else {
+      this.lastValidTransaction = lastValidTransaction;
+    }
   }
 
   /** @category 6. Meta */
   get headerMeta(): Meta {
-    return this.core.header.meta as Meta;
+    return this.core.verified.header.meta as Meta;
   }
 
   /** @category 6. Meta */
@@ -403,19 +443,11 @@ export class RawCoListView<
 
   /** @category 3. Subscription */
   subscribe(listener: (coList: this) => void): () => void {
-    return this.core.subscribe((content) => {
-      listener(content as this);
+    return this.core.subscribe((core) => {
+      listener(core.getCurrentContent() as this);
     });
   }
-}
 
-export class RawCoList<
-    Item extends JsonValue = JsonValue,
-    Meta extends JsonObject | null = JsonObject | null,
-  >
-  extends RawCoListView<Item, Meta>
-  implements RawCoValue
-{
   /** Appends `item` after the item currently at index `after`.
    *
    * If `privacy` is `"private"` **(default)**, `item` is encrypted in the transaction, only readable by other members of the group this `CoList` belongs to. Not even sync servers can see the content in plaintext.
@@ -480,8 +512,7 @@ export class RawCoList<
     }
 
     this.core.makeTransaction(changes, privacy);
-
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /**
@@ -528,7 +559,7 @@ export class RawCoList<
       privacy,
     );
 
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /** Deletes the item at index `at`.
@@ -555,7 +586,7 @@ export class RawCoList<
       privacy,
     );
 
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   replace(
@@ -583,7 +614,7 @@ export class RawCoList<
       ],
       privacy,
     );
-    this.rebuildFromCore();
+    this.processNewTransactions();
   }
 
   /** @internal */
@@ -593,6 +624,9 @@ export class RawCoList<
     this.afterStart = listAfter.afterStart;
     this.beforeEnd = listAfter.beforeEnd;
     this.insertions = listAfter.insertions;
+    this.totalValidTransactions = listAfter.totalValidTransactions;
+    this.lastValidTransaction = listAfter.lastValidTransaction;
+    this.knownTransactions = listAfter.knownTransactions;
     this.deletionsByInsertion = listAfter.deletionsByInsertion;
     this._cachedEntries = undefined;
   }

package/src/coValues/coMap.ts

@@ -1,5 +1,8 @@
 import { CoID, RawCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
+import {
+  AvailableCoValueCore,
+  CoValueCore,
+} from "../coValueCore/coValueCore.js";
 import { AgentID, TransactionID } from "../ids.js";
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { CoValueKnownState } from "../sync.js";
@@ -39,7 +42,7 @@ export class RawCoMapView<
   /** @category 6. Meta */
   type = "comap" as const;
   /** @category 6. Meta */
-  core: CoValueCore;
+  core: AvailableCoValueCore;
   /** @internal */
   latest: {
     [Key in keyof Shape & string]?: MapOp<Key, Shape[Key]>;
@@ -60,9 +63,11 @@ export class RawCoMapView<
   /** @category 6. Meta */
   readonly _shape!: Shape;
 
+  totalValidTransactions = 0;
+
   /** @internal */
   constructor(
-    core: CoValueCore,
+    core: AvailableCoValueCore,
     options?: {
       ignorePrivateTransactions: boolean;
     },
@@ -133,6 +138,8 @@ export class RawCoMapView<
       }
     }
 
+    this.totalValidTransactions += newValidTransactions.length;
+
     for (const entries of changedEntries.values()) {
       entries.sort(this.core.compareTransactions);
     }
@@ -148,7 +155,7 @@ export class RawCoMapView<
 
   /** @category 6. Meta */
   get headerMeta(): Meta {
-    return this.core.header.meta as Meta;
+    return this.core.verified.header.meta as Meta;
   }
 
   /** @category 6. Meta */
@@ -340,8 +347,8 @@ export class RawCoMapView<
 
   /** @category 3. Subscription */
   subscribe(listener: (coMap: this) => void): () => void {
-    return this.core.subscribe((content) => {
-      listener(content as this);
+    return this.core.subscribe((core) => {
+      listener(core.getCurrentContent() as this);
     });
   }
 }