cojson 0.13.17 → 0.13.20

package/src/{coValueCore.ts → coValueCore/coValueCore.ts}

@@ -1,8 +1,10 @@
-import { Result, err, ok } from "neverthrow";
-import { AnyRawCoValue, RawCoValue } from "./coValue.js";
-import { ControlledAccountOrAgent, RawAccountID } from "./coValues/account.js";
-import { RawGroup } from "./coValues/group.js";
-import { coreToCoValue } from "./coreToCoValue.js";
+import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
+import { Result, err } from "neverthrow";
+import { PeerState } from "../PeerState.js";
+import { RawCoValue } from "../coValue.js";
+import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
+import { RawGroup } from "../coValues/group.js";
+import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -12,7 +14,7 @@ import {
   Signature,
   SignerID,
   StreamingHash,
-} from "./crypto/crypto.js";
+} from "../crypto/crypto.js";
 import {
   RawCoID,
   SessionID,
@@ -20,21 +22,20 @@ import {
   getGroupDependentKeyList,
   getParentGroupId,
   isParentGroupReference,
-} from "./ids.js";
-import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
-import { JsonObject, JsonValue } from "./jsonValue.js";
-import { LocalNode, ResolveAccountAgentError } from "./localNode.js";
-import { logger } from "./logger.js";
+} from "../ids.js";
+import { parseJSON, stableStringify } from "../jsonStringify.js";
+import { JsonValue } from "../jsonValue.js";
+import { LocalNode, ResolveAccountAgentError } from "../localNode.js";
+import { logger } from "../logger.js";
 import {
-  PermissionsDef as RulesetDef,
   determineValidTransactions,
   isKeyForKeyField,
-} from "./permissions.js";
-import { getPriorityFromHeader } from "./priority.js";
-import { CoValueKnownState, NewContentMessage } from "./sync.js";
-import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
-import { expectGroup } from "./typeUtils/expectGroup.js";
-import { isAccountID } from "./typeUtils/isAccountID.js";
+} from "../permissions.js";
+import { CoValueKnownState, PeerID, emptyKnownState } from "../sync.js";
+import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
+import { expectGroup } from "../typeUtils/expectGroup.js";
+import { isAccountID } from "../typeUtils/isAccountID.js";
+import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 
 /**
     In order to not block other concurrently syncing CoValues we introduce a maximum size of transactions,
@@ -45,17 +46,6 @@ import { isAccountID } from "./typeUtils/isAccountID.js";
 **/
 export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
 
-export type CoValueHeader = {
-  type: AnyRawCoValue["type"];
-  ruleset: RulesetDef;
-  meta: JsonObject | null;
-} & CoValueUniqueness;
-
-export type CoValueUniqueness = {
-  uniqueness: JsonValue;
-  createdAt?: `2${string}` | null;
-};
-
 export function idforHeader(
   header: CoValueHeader,
   crypto: CryptoProvider,
@@ -64,29 +54,6 @@ export function idforHeader(
   return `co_z${hash.slice("shortHash_z".length)}`;
 }
 
-type SessionLog = {
-  transactions: Transaction[];
-  lastHash?: Hash;
-  streamingHash: StreamingHash;
-  signatureAfter: { [txIdx: number]: Signature | undefined };
-  lastSignature: Signature;
-};
-
-export type PrivateTransaction = {
-  privacy: "private";
-  madeAt: number;
-  keyUsed: KeyID;
-  encryptedChanges: Encrypted<JsonValue[], { in: RawCoID; tx: TransactionID }>;
-};
-
-export type TrustingTransaction = {
-  privacy: "trusting";
-  madeAt: number;
-  changes: Stringified<JsonValue[]>;
-};
-
-export type Transaction = PrivateTransaction | TrustingTransaction;
-
 export type DecryptedTransaction = {
   txID: TransactionID;
   changes: JsonValue[];
@@ -95,55 +62,246 @@ export type DecryptedTransaction = {
 
 const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
 
+export type AvailableCoValueCore = CoValueCore & { verified: VerifiedState };
+
+export const CO_VALUE_LOADING_CONFIG = {
+  MAX_RETRIES: 2,
+  TIMEOUT: 30_000,
+};
+
 export class CoValueCore {
+  // context
+  readonly node: LocalNode;
+  private readonly crypto: CryptoProvider;
+
+  // state
   id: RawCoID;
-  node: LocalNode;
-  crypto: CryptoProvider;
-  header: CoValueHeader;
-  _sessionLogs: Map<SessionID, SessionLog>;
-  _cachedContent?: RawCoValue;
-  listeners: Set<(content?: RawCoValue) => void> = new Set();
-  _decryptionCache: {
+  private _verified: VerifiedState | null;
+  /** Holds the fundamental syncable content of a CoValue,
+   * consisting of the header (verified by hash -> RawCoID)
+   * and the sessions (verified by signature).
+   *
+   * It does not do any *validation* or *decryption* and as such doesn't
+   * depend on other CoValues or the LocalNode.
+   *
+   * `CoValueCore.verified` may be null when a CoValue is requested to be
+   * loaded but no content has been received from storage or peers yet.
+   * In this case, it acts as a centralised entry to keep track of peer loading
+   * state and to subscribe to its content when it does become available. */
+  get verified() {
+    return this._verified;
+  }
+  private readonly peers = new Map<
+    PeerID,
+    | { type: "unknown" | "pending" | "available" | "unavailable" }
+    | {
+        type: "errored";
+        error: TryAddTransactionsError;
+      }
+  >();
+
+  // cached state and listeners
+  private _cachedContent?: RawCoValue;
+  private readonly listeners: Set<
+    (core: CoValueCore, unsub: () => void) => void
+  > = new Set();
+  private readonly _decryptionCache: {
     [key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined;
   } = {};
-  _cachedKnownState?: CoValueKnownState;
-  _cachedDependentOn?: RawCoID[];
-  _cachedNewContentSinceEmpty?: NewContentMessage[] | undefined;
-  _currentAsyncAddTransaction?: Promise<void>;
+  private _cachedDependentOn?: RawCoID[];
+  private counter: UpDownCounter;
 
-  constructor(
-    header: CoValueHeader,
+  private constructor(
+    init: { header: CoValueHeader } | { id: RawCoID },
     node: LocalNode,
-    internalInitSessions: Map<SessionID, SessionLog> = new Map(),
   ) {
     this.crypto = node.crypto;
-    this.id = idforHeader(header, node.crypto);
-    this.header = header;
-    this._sessionLogs = internalInitSessions;
+    if ("header" in init) {
+      this.id = idforHeader(init.header, node.crypto);
+      this._verified = new VerifiedState(
+        this.id,
+        node.crypto,
+        init.header,
+        new Map(),
+      );
+    } else {
+      this.id = init.id;
+      this._verified = null;
+    }
     this.node = node;
+
+    this.counter = metrics
+      .getMeter("cojson")
+      .createUpDownCounter("jazz.covalues.loaded", {
+        description: "The number of covalues in the system",
+        unit: "covalue",
+        valueType: ValueType.INT,
+      });
+
+    this.updateCounter(null);
+  }
+
+  static fromID(id: RawCoID, node: LocalNode): CoValueCore {
+    return new CoValueCore({ id }, node);
+  }
+
+  static fromHeader(
+    header: CoValueHeader,
+    node: LocalNode,
+  ): AvailableCoValueCore {
+    return new CoValueCore({ header }, node) as AvailableCoValueCore;
+  }
+
+  get loadingState() {
+    if (this.verified) {
+      return "available";
+    } else if (this.peers.size === 0) {
+      return "unknown";
+    }
+
+    for (const peer of this.peers.values()) {
+      if (peer.type === "pending") {
+        return "loading";
+      } else if (peer.type === "unknown") {
+        return "unknown";
+      }
+    }
+
+    return "unavailable";
+  }
+
+  isAvailable(): this is AvailableCoValueCore {
+    return !!this.verified;
+  }
+
+  isErroredInPeer(peerId: PeerID) {
+    return this.peers.get(peerId)?.type === "errored";
+  }
+
+  waitForAvailableOrUnavailable(): Promise<CoValueCore> {
+    return new Promise<CoValueCore>((resolve) => {
+      const listener = (core: CoValueCore) => {
+        if (core.isAvailable() || core.loadingState === "unavailable") {
+          resolve(core);
+          this.listeners.delete(listener);
+        }
+      };
+
+      this.listeners.add(listener);
+      listener(this);
+    });
+  }
+
+  getStateForPeer(peerId: PeerID) {
+    return this.peers.get(peerId);
+  }
+
+  private updateCounter(previousState: string | null) {
+    const newState = this.loadingState;
+
+    if (previousState !== newState) {
+      if (previousState) {
+        this.counter.add(-1, { state: previousState });
+      }
+      this.counter.add(1, { state: newState });
+    }
+  }
+
+  markNotFoundInPeer(peerId: PeerID) {
+    const previousState = this.loadingState;
+    this.peers.set(peerId, { type: "unavailable" });
+    this.updateCounter(previousState);
+    this.notifyUpdate("immediate");
+  }
+
+  // TODO: rename to "provided"
+  markAvailable(header: CoValueHeader, fromPeerId: PeerID) {
+    const previousState = this.loadingState;
+
+    if (this._verified?.sessions.size) {
+      throw new Error(
+        "CoValueCore: markAvailable called on coValue with verified sessions present!",
+      );
+    }
+    this._verified = new VerifiedState(
+      this.id,
+      this.node.crypto,
+      header,
+      new Map(),
+    );
+
+    this.peers.set(fromPeerId, { type: "available" });
+    this.updateCounter(previousState);
+    this.notifyUpdate("immediate");
+  }
+
+  internalMarkMagicallyAvailable(
+    verified: VerifiedState,
+    { forceOverwrite = false }: { forceOverwrite?: boolean } = {},
+  ) {
+    const previousState = this.loadingState;
+    this.internalShamefullyCloneVerifiedStateFrom(verified, {
+      forceOverwrite,
+    });
+    this.updateCounter(previousState);
+    this.notifyUpdate("immediate");
+  }
+
+  markErrored(peerId: PeerID, error: TryAddTransactionsError) {
+    const previousState = this.loadingState;
+    this.peers.set(peerId, { type: "errored", error });
+    this.updateCounter(previousState);
+    this.notifyUpdate("immediate");
+  }
+
+  private markPending(peerId: PeerID) {
+    const previousState = this.loadingState;
+    this.peers.set(peerId, { type: "pending" });
+    this.updateCounter(previousState);
+    this.notifyUpdate("immediate");
+  }
+
+  internalShamefullyCloneVerifiedStateFrom(
+    state: VerifiedState,
+    { forceOverwrite = false }: { forceOverwrite?: boolean } = {},
+  ) {
+    if (!forceOverwrite && this._verified?.sessions.size) {
+      throw new Error(
+        "CoValueCore: internalShamefullyCloneVerifiedStateFrom called on coValue with verified sessions present!",
+      );
+    }
+    this._verified = state.clone();
+    this._cachedContent = undefined;
+    this._cachedDependentOn = undefined;
+  }
+
+  internalShamefullyResetCachedContent() {
+    this._cachedContent = undefined;
+    this._cachedDependentOn = undefined;
   }
 
   groupInvalidationSubscription?: () => void;
 
   subscribeToGroupInvalidation() {
+    if (!this.verified) {
+      return;
+    }
+
     if (this.groupInvalidationSubscription) {
       return;
     }
 
-    const header = this.header;
+    const header = this.verified.header;
 
     if (header.ruleset.type == "ownedByGroup") {
       const groupId = header.ruleset.group;
-      const entry = this.node.coValuesStore.get(groupId);
+      const entry = this.node.getCoValue(groupId);
 
       if (entry.isAvailable()) {
-        this.groupInvalidationSubscription = entry.core.subscribe(
-          (_groupUpdate) => {
-            this._cachedContent = undefined;
-            this.notifyUpdate("immediate");
-          },
-          false,
-        );
+        this.groupInvalidationSubscription = entry.subscribe((_groupUpdate) => {
+          this._cachedContent = undefined;
+          this.notifyUpdate("immediate");
+        }, false);
       } else {
         logger.error("CoValueCore: Owner group not available", {
           id: this.id,
@@ -153,64 +311,47 @@ export class CoValueCore {
     }
   }
 
-  get sessionLogs(): Map<SessionID, SessionLog> {
-    return this._sessionLogs;
-  }
-
-  testWithDifferentAccount(
-    account: ControlledAccountOrAgent,
-    currentSessionID: SessionID,
-  ): CoValueCore {
-    const newNode = this.node.testWithDifferentAccount(
-      account,
-      currentSessionID,
+  contentInClonedNodeWithDifferentAccount(
+    controlledAccountOrAgent: ControlledAccountOrAgent,
+  ): RawCoValue {
+    const newNode = this.node.cloneWithDifferentAccount(
+      controlledAccountOrAgent,
     );
 
-    return newNode.expectCoValueLoaded(this.id);
+    return newNode.expectCoValueLoaded(this.id).getCurrentContent();
   }
 
   knownState(): CoValueKnownState {
-    if (this._cachedKnownState) {
-      return this._cachedKnownState;
+    if (this.isAvailable()) {
+      return this.verified.knownState();
     } else {
-      const knownState = this.knownStateUncached();
-      this._cachedKnownState = knownState;
-      return knownState;
+      return emptyKnownState(this.id);
     }
   }
 
-  /** @internal */
-  knownStateUncached(): CoValueKnownState {
-    const sessions: CoValueKnownState["sessions"] = {};
-
-    for (const [sessionID, sessionLog] of this.sessionLogs.entries()) {
-      sessions[sessionID] = sessionLog.transactions.length;
-    }
-
-    return {
-      id: this.id,
-      header: true,
-      sessions,
-    };
-  }
-
   get meta(): JsonValue {
-    return this.header?.meta ?? null;
+    return this.verified?.header.meta ?? null;
   }
 
   nextTransactionID(): TransactionID {
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: nextTransactionID called on coValue without verified state",
+      );
+    }
+
     // This is an ugly hack to get a unique but stable session ID for editing the current account
     const sessionID =
-      this.header.meta?.type === "account"
+      this.verified.header.meta?.type === "account"
         ? (this.node.currentSessionID.replace(
-            this.node.account.id,
-            this.node.account.currentAgentID(),
+            this.node.getCurrentAgent().id,
+            this.node.getCurrentAgent().currentAgentID(),
           ) as SessionID)
         : this.node.currentSessionID;
 
     return {
       sessionID,
-      txIndex: this.sessionLogs.get(sessionID)?.transactions.length || 0,
+      txIndex: this.verified.sessions.get(sessionID)?.transactions.length || 0,
     };
   }
 
@@ -219,6 +360,7 @@ export class CoValueCore {
     newTransactions: Transaction[],
     givenExpectedNewHash: Hash | undefined,
     newSignature: Signature,
+    notifyMode: "immediate" | "deferred",
     skipVerify: boolean = false,
     givenNewStreamingHash?: StreamingHash,
   ): Result<true, TryAddTransactionsError> {
@@ -228,126 +370,45 @@ export class CoValueCore {
         "Expected to know signer of transaction",
       )
       .andThen((agent) => {
+        if (!this.verified) {
+          return err({
+            type: "TriedToAddTransactionsWithoutVerifiedState",
+            id: this.id,
+          } satisfies TriedToAddTransactionsWithoutVerifiedStateErrpr);
+        }
+
         const signerID = this.crypto.getAgentSignerID(agent);
 
-        if (
-          skipVerify === true &&
-          givenNewStreamingHash &&
-          givenExpectedNewHash
-        ) {
-          this.doAddTransactions(
-            sessionID,
-            newTransactions,
-            newSignature,
-            givenExpectedNewHash,
-            givenNewStreamingHash,
-            "immediate",
-          );
-        } else {
-          const { expectedNewHash, newStreamingHash } =
-            this.expectedNewHashAfter(sessionID, newTransactions);
+        const result = this.verified.tryAddTransactions(
+          sessionID,
+          signerID,
+          newTransactions,
+          givenExpectedNewHash,
+          newSignature,
+          skipVerify,
+          givenNewStreamingHash,
+        );
 
+        if (result.isOk()) {
           if (
-            givenExpectedNewHash &&
-            givenExpectedNewHash !== expectedNewHash
+            this._cachedContent &&
+            "processNewTransactions" in this._cachedContent &&
+            typeof this._cachedContent.processNewTransactions === "function"
           ) {
-            return err({
-              type: "InvalidHash",
-              id: this.id,
-              expectedNewHash,
-              givenExpectedNewHash,
-            } satisfies InvalidHashError);
+            this._cachedContent.processNewTransactions();
+          } else {
+            this._cachedContent = undefined;
           }
 
-          if (!this.crypto.verify(newSignature, expectedNewHash, signerID)) {
-            return err({
-              type: "InvalidSignature",
-              id: this.id,
-              newSignature,
-              sessionID,
-              signerID,
-            } satisfies InvalidSignatureError);
-          }
+          this._cachedDependentOn = undefined;
 
-          this.doAddTransactions(
-            sessionID,
-            newTransactions,
-            newSignature,
-            expectedNewHash,
-            newStreamingHash,
-            "immediate",
-          );
+          this.notifyUpdate(notifyMode);
         }
 
-        return ok(true as const);
+        return result;
       });
   }
 
-  private doAddTransactions(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-    newSignature: Signature,
-    expectedNewHash: Hash,
-    newStreamingHash: StreamingHash,
-    notifyMode: "immediate" | "deferred",
-  ) {
-    if (this.node.crashed) {
-      throw new Error("Trying to add transactions after node is crashed");
-    }
-    const transactions = this.sessionLogs.get(sessionID)?.transactions ?? [];
-
-    for (const tx of newTransactions) {
-      transactions.push(tx);
-    }
-
-    const signatureAfter =
-      this.sessionLogs.get(sessionID)?.signatureAfter ?? {};
-
-    const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
-      (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
-      -1,
-    );
-
-    const sizeOfTxsSinceLastInbetweenSignature = transactions
-      .slice(lastInbetweenSignatureIdx + 1)
-      .reduce(
-        (sum, tx) =>
-          sum +
-          (tx.privacy === "private"
-            ? tx.encryptedChanges.length
-            : tx.changes.length),
-        0,
-      );
-
-    if (sizeOfTxsSinceLastInbetweenSignature > MAX_RECOMMENDED_TX_SIZE) {
-      signatureAfter[transactions.length - 1] = newSignature;
-    }
-
-    this._sessionLogs.set(sessionID, {
-      transactions,
-      lastHash: expectedNewHash,
-      streamingHash: newStreamingHash,
-      lastSignature: newSignature,
-      signatureAfter: signatureAfter,
-    });
-
-    if (
-      this._cachedContent &&
-      "processNewTransactions" in this._cachedContent &&
-      typeof this._cachedContent.processNewTransactions === "function"
-    ) {
-      this._cachedContent.processNewTransactions();
-    } else {
-      this._cachedContent = undefined;
-    }
-
-    this._cachedKnownState = undefined;
-    this._cachedDependentOn = undefined;
-    this._cachedNewContentSinceEmpty = undefined;
-
-    this.notifyUpdate(notifyMode);
-  }
-
   deferredUpdates = 0;
   nextDeferredNotify: Promise<void> | undefined;
 
@@ -357,10 +418,11 @@ export class CoValueCore {
     }
 
     if (notifyMode === "immediate") {
-      const content = this.getCurrentContent();
       for (const listener of this.listeners) {
         try {
-          listener(content);
+          listener(this, () => {
+            this.listeners.delete(listener);
+          });
         } catch (e) {
           logger.error("Error in listener for coValue " + this.id, { err: e });
         }
@@ -371,10 +433,11 @@ export class CoValueCore {
           setTimeout(() => {
             this.nextDeferredNotify = undefined;
             this.deferredUpdates = 0;
-            const content = this.getCurrentContent();
             for (const listener of this.listeners) {
               try {
-                listener(content);
+                listener(this, () => {
+                  this.listeners.delete(listener);
+                });
               } catch (e) {
                 logger.error("Error in listener for coValue " + this.id, {
                   err: e,
@@ -390,13 +453,15 @@ export class CoValueCore {
   }
 
   subscribe(
-    listener: (content?: RawCoValue) => void,
+    listener: (core: CoValueCore, unsub: () => void) => void,
     immediateInvoke = true,
   ): () => void {
     this.listeners.add(listener);
 
     if (immediateInvoke) {
-      listener(this.getCurrentContent());
+      listener(this, () => {
+        this.listeners.delete(listener);
+      });
     }
 
     return () => {
@@ -404,28 +469,16 @@ export class CoValueCore {
     };
   }
 
-  expectedNewHashAfter(
-    sessionID: SessionID,
-    newTransactions: Transaction[],
-  ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
-    const streamingHash =
-      this.sessionLogs.get(sessionID)?.streamingHash.clone() ??
-      new StreamingHash(this.crypto);
-
-    for (const transaction of newTransactions) {
-      streamingHash.update(transaction);
-    }
-
-    return {
-      expectedNewHash: streamingHash.digest(),
-      newStreamingHash: streamingHash,
-    };
-  }
-
   makeTransaction(
     changes: JsonValue[],
     privacy: "private" | "trusting",
   ): boolean {
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: makeTransaction called on coValue without verified state",
+      );
+    }
+
     const madeAt = Date.now();
 
     let transaction: Transaction;
@@ -460,20 +513,18 @@ export class CoValueCore {
 
     // This is an ugly hack to get a unique but stable session ID for editing the current account
     const sessionID =
-      this.header.meta?.type === "account"
+      this.verified.header.meta?.type === "account"
         ? (this.node.currentSessionID.replace(
-            this.node.account.id,
-            this.node.account.currentAgentID(),
+            this.node.getCurrentAgent().id,
+            this.node.getCurrentAgent().currentAgentID(),
           ) as SessionID)
         : this.node.currentSessionID;
 
-    const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
-      sessionID,
-      [transaction],
-    );
+    const { expectedNewHash, newStreamingHash } =
+      this.verified.expectedNewHashAfter(sessionID, [transaction]);
 
     const signature = this.crypto.sign(
-      this.node.account.currentSignerSecret(),
+      this.node.getCurrentAgent().currentSignerSecret(),
       expectedNewHash,
     );
 
@@ -482,6 +533,7 @@ export class CoValueCore {
       [transaction],
       expectedNewHash,
       signature,
+      "immediate",
       true,
       newStreamingHash,
     )._unsafeUnwrap({ withStackTrace: true });
@@ -497,13 +549,19 @@ export class CoValueCore {
   getCurrentContent(options?: {
     ignorePrivateTransactions: true;
   }): RawCoValue {
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: getCurrentContent called on coValue without verified state",
+      );
+    }
+
     if (!options?.ignorePrivateTransactions && this._cachedContent) {
       return this._cachedContent;
     }
 
     this.subscribeToGroupInvalidation();
 
-    const newContent = coreToCoValue(this, options);
+    const newContent = coreToCoValue(this as AvailableCoValueCore, options);
 
     if (!options?.ignorePrivateTransactions) {
       this._cachedContent = newContent;
@@ -594,19 +652,28 @@ export class CoValueCore {
     a: Pick<DecryptedTransaction, "madeAt" | "txID">,
     b: Pick<DecryptedTransaction, "madeAt" | "txID">,
   ) {
-    return (
-      a.madeAt - b.madeAt ||
-      (a.txID.sessionID === b.txID.sessionID
-        ? 0
-        : a.txID.sessionID < b.txID.sessionID
-          ? -1
-          : 1) ||
-      a.txID.txIndex - b.txID.txIndex
-    );
+    if (a.madeAt !== b.madeAt) {
+      return a.madeAt - b.madeAt;
+    }
+
+    if (a.txID.sessionID === b.txID.sessionID) {
+      return a.txID.txIndex - b.txID.txIndex;
+    }
+
+    return 0;
   }
 
-  getCurrentReadKey(): { secret: KeySecret | undefined; id: KeyID } {
-    if (this.header.ruleset.type === "group") {
+  getCurrentReadKey(): {
+    secret: KeySecret | undefined;
+    id: KeyID;
+  } {
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: getCurrentReadKey called on coValue without verified state",
+      );
+    }
+
+    if (this.verified.header.ruleset.type === "group") {
       const content = expectGroup(this.getCurrentContent());
 
       const currentKeyId = content.getCurrentReadKeyId();
@@ -621,9 +688,9 @@ export class CoValueCore {
         secret: secret,
         id: currentKeyId,
       };
-    } else if (this.header.ruleset.type === "ownedByGroup") {
+    } else if (this.verified.header.ruleset.type === "ownedByGroup") {
       return this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
+        .expectCoValueLoaded(this.verified.header.ruleset.group)
         .getCurrentReadKey();
     } else {
       throw new Error(
@@ -649,19 +716,32 @@ export class CoValueCore {
   }
 
   getUncachedReadKey(keyID: KeyID): KeySecret | undefined {
-    if (this.header.ruleset.type === "group") {
-      const content = expectGroup(
-        this.getCurrentContent({ ignorePrivateTransactions: true }),
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: getUncachedReadKey called on coValue without verified state",
       );
+    }
 
+    if (this.verified.header.ruleset.type === "group") {
+      const content = expectGroup(
+        this.getCurrentContent({ ignorePrivateTransactions: true }), // to prevent recursion
+      );
       const keyForEveryone = content.get(`${keyID}_for_everyone`);
-      if (keyForEveryone) return keyForEveryone;
+      if (keyForEveryone) {
+        return keyForEveryone;
+      }
 
       // Try to find key revelation for us
-      const lookupAccountOrAgentID =
-        this.header.meta?.type === "account"
-          ? this.node.account.currentAgentID()
-          : this.node.account.id;
+      const currentAgentOrAccountID = accountOrAgentIDfromSessionID(
+        this.node.currentSessionID,
+      );
+
+      // being careful here to avoid recursion
+      const lookupAccountOrAgentID = isAccountID(currentAgentOrAccountID)
+        ? this.id === currentAgentOrAccountID
+          ? this.crypto.getAgentID(this.node.agentSecret) // in accounts, the read key is revealed for the primitive agent
+          : currentAgentOrAccountID // current account ID
+        : currentAgentOrAccountID; // current agent ID
 
       const lastReadyKeyEdit = content.lastEditAt(
         `${keyID}_for_${lookupAccountOrAgentID}`,
@@ -675,7 +755,7 @@ export class CoValueCore {
 
         const secret = this.crypto.unseal(
           lastReadyKeyEdit.value,
-          this.node.account.currentSealerSecret(),
+          this.crypto.getAgentSealerSecret(this.node.agentSecret), // being careful here to avoid recursion
           this.crypto.getAgentSealerID(revealerAgent),
           {
             in: this.id,
@@ -763,9 +843,9 @@ export class CoValueCore {
       }
 
       return undefined;
-    } else if (this.header.ruleset.type === "ownedByGroup") {
+    } else if (this.verified.header.ruleset.type === "ownedByGroup") {
       return this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
+        .expectCoValueLoaded(this.verified.header.ruleset.group)
         .getReadKey(keyID);
     } else {
       throw new Error(
@@ -797,143 +877,27 @@ export class CoValueCore {
   }
 
   getGroup(): RawGroup {
-    if (this.header.ruleset.type !== "ownedByGroup") {
+    if (!this.verified) {
+      throw new Error(
+        "CoValueCore: getGroup called on coValue without verified state",
+      );
+    }
+
+    if (this.verified.header.ruleset.type !== "ownedByGroup") {
       throw new Error("Only values owned by groups have groups");
     }
 
     return expectGroup(
       this.node
-        .expectCoValueLoaded(this.header.ruleset.group)
+        .expectCoValueLoaded(this.verified.header.ruleset.group)
         .getCurrentContent(),
     );
   }
 
   getTx(txID: TransactionID): Transaction | undefined {
-    return this.sessionLogs.get(txID.sessionID)?.transactions[txID.txIndex];
-  }
-
-  newContentSince(
-    knownState: CoValueKnownState | undefined,
-  ): NewContentMessage[] | undefined {
-    const isKnownStateEmpty = !knownState?.header && !knownState?.sessions;
-
-    if (isKnownStateEmpty && this._cachedNewContentSinceEmpty) {
-      return this._cachedNewContentSinceEmpty;
-    }
-
-    let currentPiece: NewContentMessage = {
-      action: "content",
-      id: this.id,
-      header: knownState?.header ? undefined : this.header,
-      priority: getPriorityFromHeader(this.header),
-      new: {},
-    };
-
-    const pieces = [currentPiece];
-
-    const sentState: CoValueKnownState["sessions"] = {};
-
-    let pieceSize = 0;
-
-    let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
-
-    while (sessionsTodoAgain === "first" || sessionsTodoAgain?.size || 0 > 0) {
-      if (sessionsTodoAgain === "first") {
-        sessionsTodoAgain = undefined;
-      }
-      const sessionsTodo = sessionsTodoAgain ?? this.sessionLogs.keys();
-
-      for (const sessionIDKey of sessionsTodo) {
-        const sessionID = sessionIDKey as SessionID;
-        const log = this.sessionLogs.get(sessionID)!;
-        const knownStateForSessionID = knownState?.sessions[sessionID];
-        const sentStateForSessionID = sentState[sessionID];
-        const nextKnownSignatureIdx = getNextKnownSignatureIdx(
-          log,
-          knownStateForSessionID,
-          sentStateForSessionID,
-        );
-
-        const firstNewTxIdx =
-          sentStateForSessionID ?? knownStateForSessionID ?? 0;
-        const afterLastNewTxIdx =
-          nextKnownSignatureIdx === undefined
-            ? log.transactions.length
-            : nextKnownSignatureIdx + 1;
-
-        const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
-
-        if (nNewTx === 0) {
-          sessionsTodoAgain?.delete(sessionID);
-          continue;
-        }
-
-        if (afterLastNewTxIdx < log.transactions.length) {
-          if (!sessionsTodoAgain) {
-            sessionsTodoAgain = new Set();
-          }
-          sessionsTodoAgain.add(sessionID);
-        }
-
-        const oldPieceSize = pieceSize;
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          pieceSize +=
-            tx.privacy === "private"
-              ? tx.encryptedChanges.length
-              : tx.changes.length;
-        }
-
-        if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
-          currentPiece = {
-            action: "content",
-            id: this.id,
-            header: undefined,
-            new: {},
-            priority: getPriorityFromHeader(this.header),
-          };
-          pieces.push(currentPiece);
-          pieceSize = pieceSize - oldPieceSize;
-        }
-
-        let sessionEntry = currentPiece.new[sessionID];
-        if (!sessionEntry) {
-          sessionEntry = {
-            after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
-            newTransactions: [],
-            lastSignature: "WILL_BE_REPLACED" as Signature,
-          };
-          currentPiece.new[sessionID] = sessionEntry;
-        }
-
-        for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
-          const tx = log.transactions[txIdx]!;
-          sessionEntry.newTransactions.push(tx);
-        }
-
-        sessionEntry.lastSignature =
-          nextKnownSignatureIdx === undefined
-            ? log.lastSignature!
-            : log.signatureAfter[nextKnownSignatureIdx]!;
-
-        sentState[sessionID] =
-          (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
-      }
-    }
-
-    const piecesWithContent = pieces.filter(
-      (piece) => Object.keys(piece.new).length > 0 || piece.header,
-    );
-
-    if (piecesWithContent.length === 0) {
-      return undefined;
-    }
-
-    if (isKnownStateEmpty) {
-      this._cachedNewContentSinceEmpty = piecesWithContent;
-    }
-
-    return piecesWithContent;
+    return this.verified?.sessions.get(txID.sessionID)?.transactions[
+      txID.txIndex
+    ];
   }
 
   getDependedOnCoValues(): RawCoID[] {
@@ -948,13 +912,17 @@ export class CoValueCore {
 
   /** @internal */
   getDependedOnCoValuesUncached(): RawCoID[] {
-    return this.header.ruleset.type === "group"
+    if (!this.verified) {
+      return [];
+    }
+
+    return this.verified.header.ruleset.type === "group"
       ? getGroupDependentKeyList(expectGroup(this.getCurrentContent()).keys())
-      : this.header.ruleset.type === "ownedByGroup"
+      : this.verified.header.ruleset.type === "ownedByGroup"
         ? [
-            this.header.ruleset.group,
+            this.verified.header.ruleset.group,
             ...new Set(
-              [...this.sessionLogs.keys()]
+              [...this.verified.sessions.keys()]
                 .map((sessionID) =>
                   accountOrAgentIDfromSessionID(sessionID as SessionID),
                 )
@@ -972,19 +940,101 @@ export class CoValueCore {
   }) {
     return this.node.syncManager.waitForSync(this.id, options?.timeout);
   }
-}
 
-function getNextKnownSignatureIdx(
-  log: SessionLog,
-  knownStateForSessionID?: number,
-  sentStateForSessionID?: number,
-) {
-  return Object.keys(log.signatureAfter)
-    .map(Number)
-    .sort((a, b) => a - b)
-    .find(
-      (idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1),
-    );
+  async loadFromPeers(peers: PeerState[]) {
+    if (peers.length === 0) {
+      return;
+    }
+
+    const peersToActuallyLoadFrom = [];
+    for (const peer of peers) {
+      const currentState = this.peers.get(peer.id);
+
+      if (
+        currentState?.type === "available" ||
+        currentState?.type === "pending"
+      ) {
+        continue;
+      }
+
+      if (currentState?.type === "errored") {
+        continue;
+      }
+
+      if (currentState?.type === "unavailable") {
+        if (peer.shouldRetryUnavailableCoValues()) {
+          this.markPending(peer.id);
+          peersToActuallyLoadFrom.push(peer);
+        }
+
+        continue;
+      }
+
+      if (!currentState || currentState?.type === "unknown") {
+        this.markPending(peer.id);
+        peersToActuallyLoadFrom.push(peer);
+      }
+    }
+
+    for (const peer of peersToActuallyLoadFrom) {
+      if (peer.closed) {
+        this.markNotFoundInPeer(peer.id);
+        continue;
+      }
+
+      peer.pushOutgoingMessage({
+        action: "load",
+        ...this.knownState(),
+      });
+      peer.trackLoadRequestSent(this.id);
+
+      /**
+       * Use a very long timeout for storage peers, because under pressure
+       * they may take a long time to consume the messages queue
+       *
+       * TODO: Track errors on storage and do not rely on timeout
+       */
+      const timeoutDuration =
+        peer.role === "storage"
+          ? CO_VALUE_LOADING_CONFIG.TIMEOUT * 10
+          : CO_VALUE_LOADING_CONFIG.TIMEOUT;
+
+      const waitingForPeer = new Promise<void>((resolve) => {
+        const markNotFound = () => {
+          if (this.peers.get(peer.id)?.type === "pending") {
+            logger.warn("Timeout waiting for peer to load coValue", {
+              id: this.id,
+              peerID: peer.id,
+            });
+            this.markNotFoundInPeer(peer.id);
+          }
+        };
+
+        const timeout = setTimeout(markNotFound, timeoutDuration);
+        const removeCloseListener = peer.addCloseListener(markNotFound);
+
+        const listener = (state: CoValueCore) => {
+          const peerState = state.peers.get(peer.id);
+          if (
+            state.isAvailable() || // might have become available from another peer e.g. through handleNewContent
+            peerState?.type === "available" ||
+            peerState?.type === "errored" ||
+            peerState?.type === "unavailable"
+          ) {
+            this.listeners.delete(listener);
+            removeCloseListener();
+            clearTimeout(timeout);
+            resolve();
+          }
+        };
+
+        this.listeners.add(listener);
+        listener(this);
+      });
+
+      await waitingForPeer;
+    }
+  }
 }
 
 export type InvalidHashError = {
@@ -1002,7 +1052,13 @@ export type InvalidSignatureError = {
   signerID: SignerID;
 };
 
+export type TriedToAddTransactionsWithoutVerifiedStateErrpr = {
+  type: "TriedToAddTransactionsWithoutVerifiedState";
+  id: RawCoID;
+};
+
 export type TryAddTransactionsError =
+  | TriedToAddTransactionsWithoutVerifiedStateErrpr
   | ResolveAccountAgentError
   | InvalidHashError
   | InvalidSignatureError;