cojson 0.18.34 → 0.18.36

package/src/coValueContentMessage.ts

@@ -3,9 +3,9 @@ import { TRANSACTION_CONFIG } from "./config.js";
 import { Signature } from "./crypto/crypto.js";
 import { RawCoID, SessionID } from "./ids.js";
 import { JsonValue } from "./jsonValue.js";
-import { emptyKnownState } from "./knownState.js";
+import { CoValueKnownState, emptyKnownState } from "./knownState.js";
 import { getPriorityFromHeader } from "./priority.js";
-import { NewContentMessage } from "./sync.js";
+import { NewContentMessage, SessionNewContent } from "./sync.js";
 
 export function createContentMessage(
   id: RawCoID,
@@ -104,3 +104,30 @@ export function getContenDebugInfo(msg: NewContentMessage) {
       `Session: ${sessionID} After: ${sessionNewContent.after} New: ${sessionNewContent.newTransactions.length}`,
   );
 }
+
+export function getNewTransactionsFromContentMessage(
+  content: SessionNewContent,
+  knownState: CoValueKnownState,
+  sessionID: SessionID,
+) {
+  const ourKnownTxIdx = knownState.sessions[sessionID] ?? 0;
+  const theirFirstNewTxIdx = content.after;
+
+  if (ourKnownTxIdx < theirFirstNewTxIdx) {
+    // Flagging invalid state assumption by returning undefined
+    // not throwing an error or return an error object for performance reasons
+    return undefined;
+  }
+
+  const alreadyKnownOffset = ourKnownTxIdx - theirFirstNewTxIdx;
+
+  const newTransactions = content.newTransactions.slice(alreadyKnownOffset);
+
+  return newTransactions;
+}
+
+export function getSessionEntriesFromContentMessage(
+  content: NewContentMessage,
+) {
+  return Object.entries(content.new) as [SessionID, SessionNewContent][];
+}

package/src/coValueCore/SessionMap.ts

@@ -1,4 +1,3 @@
-import { Result, err, ok } from "neverthrow";
 import { ControlledAccountOrAgent } from "../coValues/account.js";
 import type {
   CryptoProvider,
@@ -40,6 +39,9 @@ export class SessionMap {
   // Known state related properies, mutated when adding transactions to the session map
   knownState: CoValueKnownState;
   knownStateWithStreaming: CoValueKnownState | undefined;
+  // The immutable version of the known statuses, to get a different reference when the known state is updated
+  immutableKnownState: CoValueKnownState;
+  immutableKnownStateWithStreaming: CoValueKnownState | undefined;
   streamingKnownState?: KnownStateSessions;
 
   constructor(
@@ -48,6 +50,7 @@ export class SessionMap {
     streamingKnownState?: KnownStateSessions,
   ) {
     this.knownState = { id: this.id, header: true, sessions: {} };
+    this.immutableKnownState = { id: this.id, header: true, sessions: {} };
     if (streamingKnownState) {
       this.streamingKnownState = { ...streamingKnownState };
       this.knownStateWithStreaming = {
@@ -55,6 +58,9 @@ export class SessionMap {
         header: true,
         sessions: { ...streamingKnownState },
       };
+      this.immutableKnownStateWithStreaming = cloneKnownState(
+        this.knownStateWithStreaming,
+      );
     }
   }
 
@@ -86,6 +92,17 @@ export class SessionMap {
       this.knownStateWithStreaming.sessions,
       actualStreamingKnownState,
     );
+
+    this.immutableKnownStateWithStreaming = cloneKnownState(
+      this.knownStateWithStreaming,
+    );
+  }
+
+  updateImmutableKnownState() {
+    this.immutableKnownState = cloneKnownState(this.knownState);
+    this.immutableKnownStateWithStreaming = this.knownStateWithStreaming
+      ? cloneKnownState(this.knownStateWithStreaming)
+      : undefined;
   }
 
   get(sessionID: SessionID): SessionLog | undefined {
@@ -120,24 +137,12 @@ export class SessionMap {
     newTransactions: Transaction[],
     newSignature: Signature,
     skipVerify: boolean = false,
-  ): Result<true, TryAddTransactionsError> {
+  ) {
     const sessionLog = this.getOrCreateSessionLog(sessionID, signerID);
 
-    try {
-      sessionLog.impl.tryAdd(newTransactions, newSignature, skipVerify);
-
-      this.addTransactionsToJsLog(sessionLog, newTransactions, newSignature);
+    sessionLog.impl.tryAdd(newTransactions, newSignature, skipVerify);
 
-      return ok(true as const);
-    } catch (e) {
-      return err({
-        type: "InvalidSignature",
-        id: this.id,
-        sessionID,
-        newSignature,
-        signerID,
-      } satisfies TryAddTransactionsError);
-    }
+    this.addTransactionsToJsLog(sessionLog, newTransactions, newSignature);
   }
 
   makeNewPrivateTransaction(
@@ -250,6 +255,8 @@ export class SessionMap {
         transactionsCount,
       );
     }
+
+    this.updateImmutableKnownState();
   }
 
   decryptTransaction(

package/src/coValueCore/coValueCore.ts

@@ -1,5 +1,4 @@
 import { UpDownCounter, ValueType, metrics } from "@opentelemetry/api";
-import { Result, err, ok } from "neverthrow";
 import type { PeerState } from "../PeerState.js";
 import type { RawCoValue } from "../coValue.js";
 import type { ControlledAccountOrAgent } from "../coValues/account.js";
@@ -201,7 +200,7 @@ export class CoValueCore {
       }
     | {
         type: "errored";
-        error: TryAddTransactionsError;
+        error: unknown;
       }
   >();
 
@@ -474,12 +473,10 @@ export class CoValueCore {
       new SessionMap(this.id, this.node.crypto, streamingKnownState),
     );
 
-    this.resetKnownStateCache();
-
     return true;
   }
 
-  markErrored(peerId: PeerID, error: TryAddTransactionsError) {
+  markErrored(peerId: PeerID, error: unknown) {
     const previousState = this.loadingState;
     this.loadingStatuses.set(peerId, { type: "errored", error });
     this.updateCounter(previousState);
@@ -531,27 +528,18 @@ export class CoValueCore {
       .then((core) => core.getCurrentContent());
   }
 
-  private _cachedKnownStateWithStreaming?: CoValueKnownState;
   /**
    * Returns the known state considering the known state of the streaming source
    *
    * Used to correctly manage the content & subscriptions during the content streaming process
    */
   knownStateWithStreaming(): CoValueKnownState {
-    if (this._cachedKnownStateWithStreaming) {
-      return this._cachedKnownStateWithStreaming;
-    }
-
-    const knownState = this.verified
-      ? cloneKnownState(this.verified.knownStateWithStreaming())
-      : emptyKnownState(this.id);
-
-    this._cachedKnownStateWithStreaming = knownState;
-
-    return knownState;
+    return (
+      this.verified?.immutableKnownStateWithStreaming() ??
+      emptyKnownState(this.id)
+    );
   }
 
-  private _cachedKnownState?: CoValueKnownState;
   /**
    * Returns the known state of the CoValue
    *
@@ -560,17 +548,7 @@ export class CoValueCore {
    * On change the knownState is invalidated and a new object is returned.
    */
   knownState(): CoValueKnownState {
-    if (this._cachedKnownState) {
-      return this._cachedKnownState;
-    }
-
-    const knownState = this.verified
-      ? cloneKnownState(this.verified.knownState())
-      : emptyKnownState(this.id);
-
-    this._cachedKnownState = knownState;
-
-    return knownState;
+    return this.verified?.immutableKnownState() ?? emptyKnownState(this.id);
   }
 
   get meta(): JsonValue {
@@ -612,31 +590,36 @@ export class CoValueCore {
     newTransactions: Transaction[],
     newSignature: Signature,
     skipVerify: boolean = false,
-  ): Result<true, TryAddTransactionsError> {
-    let result: Result<SignerID | undefined, TryAddTransactionsError>;
+  ) {
+    let signerID: SignerID | undefined;
 
-    if (skipVerify) {
-      result = ok(undefined);
-    } else {
-      result = this.node
-        .resolveAccountAgent(
-          accountOrAgentIDfromSessionID(sessionID),
-          "Expected to know signer of transaction",
-        )
-        .andThen((agent) => {
-          return ok(this.crypto.getAgentSignerID(agent));
-        });
-    }
+    if (!skipVerify) {
+      const result = this.node.resolveAccountAgent(
+        accountOrAgentIDfromSessionID(sessionID),
+        "Expected to know signer of transaction",
+      );
 
-    return result.andThen((signerID) => {
-      if (!this.verified) {
-        return err({
-          type: "TriedToAddTransactionsWithoutVerifiedState",
+      if (result.error || !result.value) {
+        return {
+          type: "ResolveAccountAgentError",
           id: this.id,
-        } satisfies TriedToAddTransactionsWithoutVerifiedStateErrpr);
+          error: result.error,
+        } as const;
       }
 
-      const result = this.verified.tryAddTransactions(
+      signerID = this.crypto.getAgentSignerID(result.value);
+    }
+
+    if (!this.verified) {
+      return {
+        type: "TriedToAddTransactionsWithoutVerifiedState",
+        id: this.id,
+        error: undefined,
+      };
+    }
+
+    try {
+      this.verified.tryAddTransactions(
         sessionID,
         signerID,
         newTransactions,
@@ -644,19 +627,11 @@ export class CoValueCore {
         skipVerify,
       );
 
-      if (result.isOk()) {
-        this.resetKnownStateCache();
-        this.processNewTransactions();
-        this.scheduleNotifyUpdate();
-      }
-
-      return result;
-    });
-  }
-
-  private resetKnownStateCache() {
-    this._cachedKnownState = undefined;
-    this._cachedKnownStateWithStreaming = undefined;
+      this.processNewTransactions();
+      this.scheduleNotifyUpdate();
+    } catch (e) {
+      return { type: "InvalidSignature", id: this.id, error: e } as const;
+    }
   }
 
   private processNewTransactions() {
@@ -805,7 +780,6 @@ export class CoValueCore {
 
     this.node.syncManager.recordTransactionsSize([transaction], "local");
 
-    this.resetKnownStateCache();
     this.processNewTransactions();
     this.addDependenciesFromNewTransaction(transaction);
 

package/src/coValueCore/verifiedState.ts

@@ -1,4 +1,3 @@
-import { Result, err, ok } from "neverthrow";
 import { AnyRawCoValue } from "../coValue.js";
 import {
   createContentMessage,
@@ -94,16 +93,14 @@ export class VerifiedState {
     newTransactions: Transaction[],
     newSignature: Signature,
     skipVerify: boolean = false,
-  ): Result<true, TryAddTransactionsError> {
-    const result = this.sessions.addTransaction(
+  ) {
+    this.sessions.addTransaction(
       sessionID,
       signerID,
       newTransactions,
       newSignature,
       skipVerify,
     );
-
-    return result;
   }
 
   makeNewTrustingTransaction(
@@ -293,16 +290,18 @@ export class VerifiedState {
     );
 
     if (piecesWithContent.length > 1 || this.isStreaming()) {
-      // Flag that more content is coming
-      if (knownState) {
-        firstPiece.expectContentUntil = getKnownStateToSend(
-          this.knownStateWithStreaming().sessions,
-          knownState.sessions,
-        );
-      } else {
-        firstPiece.expectContentUntil = {
-          ...this.knownStateWithStreaming().sessions,
-        };
+      if (!opts?.skipExpectContentUntil) {
+        // Flag that more content is coming
+        if (knownState) {
+          firstPiece.expectContentUntil = getKnownStateToSend(
+            this.knownStateWithStreaming().sessions,
+            knownState.sessions,
+          );
+        } else {
+          firstPiece.expectContentUntil = {
+            ...this.knownStateWithStreaming().sessions,
+          };
+        }
       }
     }
 
@@ -321,6 +320,17 @@ export class VerifiedState {
     return this.sessions.knownStateWithStreaming ?? this.knownState();
   }
 
+  immutableKnownState() {
+    return this.sessions.immutableKnownState;
+  }
+
+  immutableKnownStateWithStreaming() {
+    return (
+      this.sessions.immutableKnownStateWithStreaming ??
+      this.immutableKnownState()
+    );
+  }
+
   isStreaming(): boolean {
     return Boolean(this.sessions.knownStateWithStreaming);
   }

package/src/coValues/group.ts

@@ -206,7 +206,7 @@ export class RawGroup<
     }
 
     const runMigrations = () => {
-      rotateReadKeyIfNeeded(this);
+      // rotateReadKeyIfNeeded(this);
       healMissingKeyForEveryone(this);
     };
 
@@ -508,12 +508,14 @@ export class RawGroup<
         memberRole === "writerInvite" ||
         memberRole === "adminInvite"
       ) {
-        const otherMemberAgent = this.core.node
-          .resolveAccountAgent(
-            otherMemberKey,
-            "Expected member agent to be loaded",
-          )
-          ._unsafeUnwrap({ withStackTrace: true });
+        const otherMemberAgent = this.core.node.resolveAccountAgent(
+          otherMemberKey,
+          "Expected member agent to be loaded",
+        ).value;
+
+        if (!otherMemberAgent) {
+          throw new Error("Expected member agent to be loaded");
+        }
 
         this.storeKeyRevelationForMember(
           otherMemberKey,
@@ -689,9 +691,14 @@ export class RawGroup<
 
     if (lastReadyKeyEdit?.value) {
       const revealer = lastReadyKeyEdit.by;
-      const revealerAgent = core.node
-        .resolveAccountAgent(revealer, "Expected to know revealer")
-        ._unsafeUnwrap({ withStackTrace: true });
+      const revealerAgent = core.node.resolveAccountAgent(
+        revealer,
+        "Expected to know revealer",
+      ).value;
+
+      if (!revealerAgent) {
+        throw new Error("Expected to know revealer");
+      }
 
       const secret = this.crypto.unseal(
         lastReadyKeyEdit.value,
@@ -841,12 +848,14 @@ export class RawGroup<
     const newReadKey = this.crypto.newRandomKeySecret();
 
     for (const readerID of currentlyPermittedReaders) {
-      const agent = this.core.node
-        .resolveAccountAgent(
-          readerID,
-          "Expected to know currently permitted reader",
-        )
-        ._unsafeUnwrap({ withStackTrace: true });
+      const agent = this.core.node.resolveAccountAgent(
+        readerID,
+        "Expected to know currently permitted reader",
+      ).value;
+
+      if (!agent) {
+        throw new Error("Expected to know currently permitted reader");
+      }
 
       this.storeKeyRevelationForMember(
         readerID,
@@ -861,12 +870,14 @@ export class RawGroup<
      * and reveal them to the other non-writeOnly members
      */
     for (const writeOnlyMemberID of writeOnlyMembers) {
-      const agent = this.core.node
-        .resolveAccountAgent(
-          writeOnlyMemberID,
-          "Expected to know writeOnly member",
-        )
-        ._unsafeUnwrap({ withStackTrace: true });
+      const agent = this.core.node.resolveAccountAgent(
+        writeOnlyMemberID,
+        "Expected to know writeOnly member",
+      ).value;
+
+      if (!agent) {
+        throw new Error("Expected to know writeOnly member");
+      }
 
       const writeOnlyKey = this.crypto.newRandomKeySecret();
 
@@ -879,12 +890,14 @@ export class RawGroup<
       this.set(`writeKeyFor_${writeOnlyMemberID}`, writeOnlyKey.id, "trusting");
 
       for (const readerID of currentlyPermittedReaders) {
-        const agent = this.core.node
-          .resolveAccountAgent(
-            readerID,
-            "Expected to know currently permitted reader",
-          )
-          ._unsafeUnwrap({ withStackTrace: true });
+        const agent = this.core.node.resolveAccountAgent(
+          readerID,
+          "Expected to know currently permitted reader",
+        ).value;
+
+        if (!agent) {
+          throw new Error("Expected to know currently permitted reader");
+        }
 
         this.storeKeyRevelationForMember(
           readerID,

package/src/knownState.ts

@@ -113,10 +113,7 @@ export function areCurrentSessionsInSyncWith(
   current: Record<string, number>,
   target: Record<string, number>,
 ) {
-  for (const [sessionId, currentCount] of Object.entries(current) as [
-    SessionID,
-    number,
-  ][]) {
+  for (const [sessionId, currentCount] of Object.entries(current)) {
     const targetCount = target[sessionId] ?? 0;
     if (currentCount !== targetCount) {
       return false;
@@ -133,10 +130,7 @@ export function isKnownStateSubsetOf(
   current: Record<string, number>,
   target: Record<string, number>,
 ) {
-  for (const [sessionId, currentCount] of Object.entries(current) as [
-    SessionID,
-    number,
-  ][]) {
+  for (const [sessionId, currentCount] of Object.entries(current)) {
     const targetCount = target[sessionId] ?? 0;
     if (currentCount > targetCount) {
       return false;
@@ -154,10 +148,7 @@ export function getKnownStateToSend(
   target: Record<string, number>,
 ) {
   const toSend: Record<string, number> = {};
-  for (const [sessionId, currentCount] of Object.entries(current) as [
-    SessionID,
-    number,
-  ][]) {
+  for (const [sessionId, currentCount] of Object.entries(current)) {
     const targetCount = target[sessionId] ?? 0;
     if (currentCount > targetCount) {
       toSend[sessionId] = currentCount;

package/src/localNode.ts

@@ -1,4 +1,3 @@
-import { Result, err, ok } from "neverthrow";
 import { GarbageCollector } from "./GarbageCollector.js";
 import type { CoID } from "./coValue.js";
 import type { RawCoValue } from "./coValue.js";
@@ -689,12 +688,9 @@ export class LocalNode {
   }
 
   /** @internal */
-  resolveAccountAgent(
-    id: RawAccountID | AgentID,
-    expectation?: string,
-  ): Result<AgentID, ResolveAccountAgentError> {
+  resolveAccountAgent(id: RawAccountID | AgentID, expectation?: string) {
     if (isAgentID(id)) {
-      return ok(id);
+      return { value: id, error: undefined };
     }
 
     let coValue: AvailableCoValueCore;
@@ -702,12 +698,7 @@ export class LocalNode {
     try {
       coValue = this.expectCoValueLoaded(id, expectation);
     } catch (e) {
-      return err({
-        type: "ErrorLoadingCoValueCore",
-        expectation,
-        id,
-        error: e,
-      } satisfies LoadCoValueCoreError);
+      return { value: undefined, error: e };
     }
 
     if (
@@ -717,14 +708,15 @@ export class LocalNode {
       !("type" in coValue.verified.header.meta) ||
       coValue.verified.header.meta.type !== "account"
     ) {
-      return err({
-        type: "UnexpectedlyNotAccount",
-        expectation,
-        id,
-      } satisfies UnexpectedlyNotAccountError);
+      return {
+        value: undefined,
+        error: new Error(`Unexpectedly not account: ${expectation}`),
+      };
     }
 
-    return ok((coValue.getCurrentContent() as RawAccount).currentAgentID());
+    const account = coValue.getCurrentContent() as RawAccount;
+
+    return { value: account.currentAgentID(), error: undefined };
   }
 
   createGroup(

package/src/sync.ts

@@ -4,6 +4,8 @@ import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import {
   getContenDebugInfo,
+  getNewTransactionsFromContentMessage,
+  getSessionEntriesFromContentMessage,
   getTransactionSize,
   knownStateFromContent,
 } from "./coValueContentMessage.js";
@@ -609,56 +611,55 @@ export class SyncManager {
     /**
      * The coValue is in memory, load the transactions from the content message
      */
-    for (const [sessionID, newContentForSession] of Object.entries(msg.new) as [
-      SessionID,
-      SessionNewContent,
-    ][]) {
-      const ourKnownTxIdx =
-        coValue.verified.sessions.get(sessionID)?.transactions.length;
-      const theirFirstNewTxIdx = newContentForSession.after;
-
-      if ((ourKnownTxIdx || 0) < theirFirstNewTxIdx) {
+    for (const [
+      sessionID,
+      newContentForSession,
+    ] of getSessionEntriesFromContentMessage(msg)) {
+      const newTransactions = getNewTransactionsFromContentMessage(
+        newContentForSession,
+        coValue.knownState(),
+        sessionID,
+      );
+
+      if (newTransactions === undefined) {
         invalidStateAssumed = true;
         continue;
       }
 
-      const alreadyKnownOffset = ourKnownTxIdx
-        ? ourKnownTxIdx - theirFirstNewTxIdx
-        : 0;
-
-      const newTransactions =
-        newContentForSession.newTransactions.slice(alreadyKnownOffset);
-
       if (newTransactions.length === 0) {
         continue;
       }
 
       // TODO: Handle invalid signatures in the middle of streaming
       // This could cause a situation where we are unable to load a chunk, and ask for a correction for all the subsequent chunks
-      const result = coValue.tryAddTransactions(
+      const error = coValue.tryAddTransactions(
         sessionID,
         newTransactions,
         newContentForSession.lastSignature,
         this.skipVerify,
       );
 
-      if (result.isErr()) {
+      if (error) {
         if (peer) {
           logger.error("Failed to add transactions", {
             peerId: peer.id,
             peerRole: peer.role,
             id: msg.id,
-            err: result.error,
+            errorType: error.type,
+            err: error.error,
+            sessionID,
             msgKnownState: knownStateFromContent(msg).sessions,
+            msgSummary: getContenDebugInfo(msg),
             knownState: coValue.knownState().sessions,
-            newContent: validNewContent.new,
           });
           // TODO Mark only the session as errored, not the whole coValue
-          coValue.markErrored(peer.id, result.error);
+          coValue.markErrored(peer.id, error);
         } else {
           logger.error("Failed to add transactions from storage", {
             id: msg.id,
-            err: result.error,
+            err: error.error,
+            sessionID,
+            errorType: error.type,
           });
         }
         continue;
@@ -669,9 +670,7 @@ export class SyncManager {
       }
 
       // The new content for this session has been verified, so we can store it
-      if (result.value) {
-        validNewContent.new[sessionID] = newContentForSession;
-      }
+      validNewContent.new[sessionID] = newContentForSession;
     }
 
     if (peer) {