cojson 0.15.7 → 0.15.9

package/src/coValueCore/coValueCore.ts

@@ -4,6 +4,7 @@ import { PeerState } from "../PeerState.js";
 import { RawCoValue } from "../coValue.js";
 import { ControlledAccountOrAgent, RawAccountID } from "../coValues/account.js";
 import { RawGroup } from "../coValues/group.js";
+import { CO_VALUE_LOADING_CONFIG, MAX_RECOMMENDED_TX_SIZE } from "../config.js";
 import { coreToCoValue } from "../coreToCoValue.js";
 import {
   CryptoProvider,
@@ -19,8 +20,6 @@ import {
   RawCoID,
   SessionID,
   TransactionID,
-  getGroupDependentKey,
-  getGroupDependentKeyList,
   getParentGroupId,
   isParentGroupReference,
 } from "../ids.js";
@@ -39,15 +38,6 @@ import { isAccountID } from "../typeUtils/isAccountID.js";
 import { getDependedOnCoValuesFromRawData } from "./utils.js";
 import { CoValueHeader, Transaction, VerifiedState } from "./verifiedState.js";
 
-/**
-    In order to not block other concurrently syncing CoValues we introduce a maximum size of transactions,
-    since they are the smallest unit of progress that can be synced within a CoValue.
-    This is particularly important for storing binary data in CoValues, since they are likely to be at least on the order of megabytes.
-    This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
-    to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
-**/
-export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
-
 export function idforHeader(
   header: CoValueHeader,
   crypto: CryptoProvider,
@@ -66,12 +56,6 @@ const readKeyCache = new WeakMap<CoValueCore, { [id: KeyID]: KeySecret }>();
 
 export type AvailableCoValueCore = CoValueCore & { verified: VerifiedState };
 
-export const CO_VALUE_LOADING_CONFIG = {
-  MAX_RETRIES: 1,
-  TIMEOUT: 30_000,
-  RETRY_DELAY: 3000,
-};
-
 export class CoValueCore {
   // context
   readonly node: LocalNode;
@@ -174,7 +158,11 @@ export class CoValueCore {
   }
 
   isAvailable(): this is AvailableCoValueCore {
-    return !!this.verified && this.missingDependencies.size === 0;
+    return this.hasVerifiedContent() && this.missingDependencies.size === 0;
+  }
+
+  hasVerifiedContent(): this is AvailableCoValueCore {
+    return !!this.verified;
   }
 
   isErroredInPeer(peerId: PeerID) {
@@ -232,12 +220,41 @@ export class CoValueCore {
   }
 
   missingDependencies = new Set<RawCoID>();
+
+  // Checks if the current CoValueCore is already a missing dependency of the given CoValueCore
+  checkCircularDependencies(dependency: CoValueCore) {
+    const visited = new Set<RawCoID>();
+    const stack = [dependency];
+
+    while (stack.length > 0) {
+      const current = stack.pop();
+
+      if (!current) {
+        return true;
+      }
+
+      visited.add(current.id);
+
+      for (const dependency of current.missingDependencies) {
+        if (dependency === this.id) {
+          return false;
+        }
+
+        if (!visited.has(dependency)) {
+          stack.push(this.node.getCoValue(dependency));
+        }
+      }
+    }
+
+    return true;
+  }
+
   markMissingDependency(dependency: RawCoID) {
     const value = this.node.getCoValue(dependency);
 
     if (value.isAvailable()) {
       this.missingDependencies.delete(dependency);
-    } else {
+    } else if (this.checkCircularDependencies(value)) {
       const unsubscribe = value.subscribe(() => {
         if (value.isAvailable()) {
           this.missingDependencies.delete(dependency);
@@ -253,7 +270,11 @@ export class CoValueCore {
     }
   }
 
-  provideHeader(header: CoValueHeader, fromPeerId: PeerID) {
+  provideHeader(
+    header: CoValueHeader,
+    fromPeerId: PeerID,
+    streamingKnownState?: CoValueKnownState["sessions"],
+  ) {
     const previousState = this.loadingState;
 
     if (this._verified?.sessions.size) {
@@ -266,9 +287,11 @@ export class CoValueCore {
       this.node.crypto,
       header,
       new Map(),
+      streamingKnownState,
     );
 
     this.peers.set(fromPeerId, { type: "available" });
+
     this.updateCounter(previousState);
     this.notifyUpdate("immediate");
   }
@@ -292,7 +315,7 @@ export class CoValueCore {
     this.notifyUpdate("immediate");
   }
 
-  private markPending(peerId: PeerID) {
+  markPending(peerId: PeerID) {
     const previousState = this.loadingState;
     this.peers.set(peerId, { type: "pending" });
     this.updateCounter(previousState);
@@ -355,6 +378,14 @@ export class CoValueCore {
       .getCurrentContent();
   }
 
+  knownStateWithStreaming(): CoValueKnownState {
+    if (this.isAvailable()) {
+      return this.verified.knownStateWithStreaming();
+    } else {
+      return emptyKnownState(this.id);
+    }
+  }
+
   knownState(): CoValueKnownState {
     if (this.isAvailable()) {
       return this.verified.knownState();
@@ -980,65 +1011,68 @@ export class CoValueCore {
     return this.node.syncManager.waitForSync(this.id, options?.timeout);
   }
 
-  async loadFromPeers(peers: PeerState[]) {
-    if (peers.length === 0) {
+  load(peers: PeerState[]) {
+    this.loadFromStorage((found) => {
+      // When found the load is triggered by handleNewContent
+      if (!found) {
+        this.loadFromPeers(peers);
+      }
+    });
+  }
+
+  loadFromStorage(done?: (found: boolean) => void) {
+    const node = this.node;
+
+    if (!node.storage) {
+      done?.(false);
       return;
     }
 
-    const peersToActuallyLoadFrom = {
-      storage: [] as PeerState[],
-      server: [] as PeerState[],
-    };
+    const currentState = this.peers.get("storage");
 
-    for (const peer of peers) {
-      const currentState = this.peers.get(peer.id);
+    if (currentState && currentState.type !== "unknown") {
+      done?.(currentState.type === "available");
+      return;
+    }
 
-      if (
-        currentState?.type === "available" ||
-        currentState?.type === "pending"
-      ) {
-        continue;
-      }
+    this.markPending("storage");
+    node.storage.load(
+      this.id,
+      (data) => {
+        node.syncManager.handleNewContent(data, "storage");
+      },
+      (found) => {
+        if (!found) {
+          this.markNotFoundInPeer("storage");
+        }
 
-      if (currentState?.type === "errored") {
-        continue;
-      }
+        done?.(found);
+      },
+    );
+  }
 
-      if (currentState?.type === "unavailable") {
-        if (peer.role === "server") {
-          peersToActuallyLoadFrom.server.push(peer);
-          this.markPending(peer.id);
-        }
+  loadFromPeers(peers: PeerState[]) {
+    if (peers.length === 0) {
+      return;
+    }
 
-        continue;
-      }
+    const peersToActuallyLoadFrom = [] as PeerState[];
 
-      if (!currentState || currentState?.type === "unknown") {
-        if (peer.role === "storage") {
-          peersToActuallyLoadFrom.storage.push(peer);
-        } else {
-          peersToActuallyLoadFrom.server.push(peer);
-        }
+    for (const peer of peers) {
+      const currentState = this.peers.get(peer.id)?.type;
 
+      if (
+        !currentState ||
+        currentState === "unknown" ||
+        currentState === "unavailable"
+      ) {
+        peersToActuallyLoadFrom.push(peer);
         this.markPending(peer.id);
       }
     }
 
-    // Load from storage peers first, then from server peers
-    if (peersToActuallyLoadFrom.storage.length > 0) {
-      await Promise.all(
-        peersToActuallyLoadFrom.storage.map((peer) =>
-          this.internalLoadFromPeer(peer),
-        ),
-      );
-    }
-
-    if (peersToActuallyLoadFrom.server.length > 0) {
-      await Promise.all(
-        peersToActuallyLoadFrom.server.map((peer) =>
-          this.internalLoadFromPeer(peer),
-        ),
-      );
+    for (const peer of peersToActuallyLoadFrom) {
+      this.internalLoadFromPeer(peer);
     }
   }
 

package/src/coValueCore/verifiedState.ts

@@ -1,5 +1,6 @@
 import { Result, err, ok } from "neverthrow";
 import { AnyRawCoValue } from "../coValue.js";
+import { MAX_RECOMMENDED_TX_SIZE } from "../config.js";
 import {
   CryptoProvider,
   Encrypted,
@@ -15,11 +16,7 @@ import { JsonObject, JsonValue } from "../jsonValue.js";
 import { PermissionsDef as RulesetDef } from "../permissions.js";
 import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
-import {
-  InvalidHashError,
-  InvalidSignatureError,
-  MAX_RECOMMENDED_TX_SIZE,
-} from "./coValueCore.js";
+import { InvalidHashError, InvalidSignatureError } from "./coValueCore.js";
 import { TryAddTransactionsError } from "./coValueCore.js";
 
 export type CoValueHeader = {
@@ -50,8 +47,7 @@ export type Transaction = PrivateTransaction | TrustingTransaction;
 
 type SessionLog = {
   readonly transactions: Transaction[];
-  lastHash?: Hash;
-  streamingHash: StreamingHash;
+  streamingHash?: StreamingHash;
   readonly signatureAfter: { [txIdx: number]: Signature | undefined };
   lastSignature: Signature;
 };
@@ -65,17 +61,22 @@ export class VerifiedState {
   readonly sessions: ValidatedSessions;
   private _cachedKnownState?: CoValueKnownState;
   private _cachedNewContentSinceEmpty: NewContentMessage[] | undefined;
+  private streamingKnownState?: CoValueKnownState["sessions"];
 
   constructor(
     id: RawCoID,
     crypto: CryptoProvider,
     header: CoValueHeader,
     sessions: ValidatedSessions,
+    streamingKnownState?: CoValueKnownState["sessions"],
   ) {
     this.id = id;
     this.crypto = crypto;
     this.header = header;
     this.sessions = sessions;
+    this.streamingKnownState = streamingKnownState
+      ? { ...streamingKnownState }
+      : undefined;
   }
 
   clone(): VerifiedState {
@@ -84,13 +85,18 @@ export class VerifiedState {
     for (let [sessionID, sessionLog] of this.sessions) {
       clonedSessions.set(sessionID, {
         lastSignature: sessionLog.lastSignature,
-        lastHash: sessionLog.lastHash,
-        streamingHash: sessionLog.streamingHash.clone(),
+        streamingHash: sessionLog.streamingHash?.clone(),
         signatureAfter: { ...sessionLog.signatureAfter },
         transactions: sessionLog.transactions.slice(),
       } satisfies SessionLog);
     }
-    return new VerifiedState(this.id, this.crypto, this.header, clonedSessions);
+    return new VerifiedState(
+      this.id,
+      this.crypto,
+      this.header,
+      clonedSessions,
+      this.streamingKnownState,
+    );
   }
 
   tryAddTransactions(
@@ -102,12 +108,11 @@ export class VerifiedState {
     skipVerify: boolean = false,
     givenNewStreamingHash?: StreamingHash,
   ): Result<true, TryAddTransactionsError> {
-    if (skipVerify === true && givenNewStreamingHash && givenExpectedNewHash) {
+    if (skipVerify === true) {
       this.doAddTransactions(
         sessionID,
         newTransactions,
         newSignature,
-        givenExpectedNewHash,
         givenNewStreamingHash,
       );
     } else {
@@ -139,7 +144,6 @@ export class VerifiedState {
         sessionID,
         newTransactions,
         newSignature,
-        expectedNewHash,
         newStreamingHash,
       );
     }
@@ -151,16 +155,16 @@ export class VerifiedState {
     sessionID: SessionID,
     newTransactions: Transaction[],
     newSignature: Signature,
-    expectedNewHash: Hash,
-    newStreamingHash: StreamingHash,
+    newStreamingHash?: StreamingHash,
   ) {
-    const transactions = this.sessions.get(sessionID)?.transactions ?? [];
+    const sessionLog = this.sessions.get(sessionID);
+    const transactions = sessionLog?.transactions ?? [];
 
     for (const tx of newTransactions) {
       transactions.push(tx);
     }
 
-    const signatureAfter = this.sessions.get(sessionID)?.signatureAfter ?? {};
+    const signatureAfter = sessionLog?.signatureAfter ?? {};
 
     const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
       (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
@@ -184,7 +188,6 @@ export class VerifiedState {
 
     this.sessions.set(sessionID, {
       transactions,
-      lastHash: expectedNewHash,
       streamingHash: newStreamingHash,
       lastSignature: newSignature,
       signatureAfter: signatureAfter,
@@ -198,9 +201,27 @@ export class VerifiedState {
     sessionID: SessionID,
     newTransactions: Transaction[],
   ): { expectedNewHash: Hash; newStreamingHash: StreamingHash } {
-    const streamingHash =
-      this.sessions.get(sessionID)?.streamingHash.clone() ??
-      new StreamingHash(this.crypto);
+    const sessionLog = this.sessions.get(sessionID);
+
+    if (!sessionLog?.streamingHash) {
+      const streamingHash = new StreamingHash(this.crypto);
+      const oldTransactions = sessionLog?.transactions ?? [];
+
+      for (const transaction of oldTransactions) {
+        streamingHash.update(transaction);
+      }
+
+      for (const transaction of newTransactions) {
+        streamingHash.update(transaction);
+      }
+
+      return {
+        expectedNewHash: streamingHash.digest(),
+        newStreamingHash: streamingHash,
+      };
+    }
+
+    const streamingHash = sessionLog.streamingHash.clone();
 
     for (const transaction of newTransactions) {
       streamingHash.update(transaction);
@@ -285,6 +306,11 @@ export class VerifiedState {
         }
 
         if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+          if (!currentPiece.expectContentUntil && pieces.length === 1) {
+            currentPiece.expectContentUntil =
+              this.knownStateWithStreaming().sessions;
+          }
+
           currentPiece = {
             action: "content",
             id: this.id,
@@ -336,6 +362,50 @@ export class VerifiedState {
     return piecesWithContent;
   }
 
+  /**
+   * Returns the known state considering the known state of the streaming source
+   *
+   * Used to correctly manage the content & subscriptions during the content streaming process
+   */
+  knownStateWithStreaming(): CoValueKnownState {
+    const knownState = this.knownState();
+
+    if (this.streamingKnownState) {
+      const newSessions: CoValueKnownState["sessions"] = {};
+      const entries = Object.entries(this.streamingKnownState);
+
+      for (const [sessionID, txs] of entries) {
+        newSessions[sessionID as SessionID] = txs;
+        if ((knownState.sessions[sessionID as SessionID] ?? 0) < txs) {
+          newSessions[sessionID as SessionID] = txs;
+        } else {
+          newSessions[sessionID as SessionID] = txs;
+          delete this.streamingKnownState[sessionID as SessionID];
+        }
+      }
+
+      if (Object.keys(this.streamingKnownState).length === 0) {
+        this.streamingKnownState = undefined;
+        return knownState;
+      } else {
+        return {
+          id: knownState.id,
+          header: knownState.header,
+          sessions: newSessions,
+        };
+      }
+    }
+
+    return knownState;
+  }
+
+  isStreaming(): boolean {
+    // Call knownStateWithStreaming to delete the streamingKnownState when it matches the current knownState
+    this.knownStateWithStreaming();
+
+    return this.streamingKnownState !== undefined;
+  }
+
   knownState(): CoValueKnownState {
     if (this._cachedKnownState) {
       return this._cachedKnownState;

package/src/coValues/group.ts

@@ -193,7 +193,7 @@ export class RawGroup<
 
   loadAllChildGroups() {
     const requests: Promise<unknown>[] = [];
-    const peers = this.core.node.syncManager.getServerAndStoragePeers();
+    const peers = this.core.node.syncManager.getServerPeers();
 
     for (const key of this.keys()) {
       if (!isChildGroupReference(key)) {
@@ -207,9 +207,7 @@ export class RawGroup<
         child.loadingState === "unknown" ||
         child.loadingState === "unavailable"
       ) {
-        child.loadFromPeers(peers).catch(() => {
-          logger.error(`Failed to load child group ${id}`);
-        });
+        child.load(peers);
       }
 
       requests.push(

package/src/config.ts

@@ -0,0 +1,26 @@
+/**
+    In order to not block other concurrently syncing CoValues we introduce a maximum size of transactions,
+    since they are the smallest unit of progress that can be synced within a CoValue.
+    This is particularly important for storing binary data in CoValues, since they are likely to be at least on the order of megabytes.
+    This also means that we want to keep signatures roughly after each MAX_RECOMMENDED_TX size chunk,
+    to be able to verify partially loaded CoValues or CoValues that are still being created (like a video live stream).
+**/
+export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
+
+export const CO_VALUE_LOADING_CONFIG = {
+  MAX_RETRIES: 1,
+  TIMEOUT: 30_000,
+  RETRY_DELAY: 3000,
+};
+
+export function setCoValueLoadingRetryDelay(delay: number) {
+  CO_VALUE_LOADING_CONFIG.RETRY_DELAY = delay;
+}
+
+export const SYNC_SCHEDULER_CONFIG = {
+  INCOMING_MESSAGES_TIME_BUDGET: 50,
+};
+
+export function setIncomingMessagesTimeBudget(budget: number) {
+  SYNC_SCHEDULER_CONFIG.INCOMING_MESSAGES_TIME_BUDGET = budget;
+}

package/src/crypto/WasmCrypto.ts

@@ -108,11 +108,13 @@ export class WasmCrypto extends CryptoProvider<Blake3State> {
   }
 
   verify(signature: Signature, message: JsonValue, id: SignerID): boolean {
-    return verify(
+    const result = verify(
       textEncoder.encode(signature),
       textEncoder.encode(stableStringify(message)),
       textEncoder.encode(id),
     );
+
+    return result;
   }
 
   newX25519StaticSecret(): Uint8Array {

package/src/exports.ts

@@ -1,11 +1,6 @@
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 import { type RawCoValue } from "./coValue.js";
-import {
-  CO_VALUE_LOADING_CONFIG,
-  CoValueCore,
-  MAX_RECOMMENDED_TX_SIZE,
-  idforHeader,
-} from "./coValueCore/coValueCore.js";
+import { CoValueCore, idforHeader } from "./coValueCore/coValueCore.js";
 import { CoValueUniqueness } from "./coValueCore/verifiedState.js";
 import {
   ControlledAccount,
@@ -42,7 +37,7 @@ import {
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 import { LocalNode } from "./localNode.js";
 import type { AccountRole, Role } from "./permissions.js";
-import { Channel, connectedPeers } from "./streamUtils.js";
+import { ConnectedPeerChannel, connectedPeers } from "./streamUtils.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 import { isAccountID } from "./typeUtils/isAccountID.js";
@@ -63,24 +58,21 @@ import type { AgentID, RawCoID, SessionID } from "./ids.js";
 import type { JsonObject, JsonValue } from "./jsonValue.js";
 import type * as Media from "./media.js";
 import { disablePermissionErrors } from "./permissions.js";
-import type {
-  IncomingSyncStream,
-  OutgoingSyncQueue,
-  Peer,
-  SyncMessage,
-} from "./sync.js";
-import {
-  DisconnectedError,
-  PingTimeoutError,
-  SyncManager,
-  emptyKnownState,
-} from "./sync.js";
+import type { Peer, SyncMessage } from "./sync.js";
+import { DisconnectedError, SyncManager, emptyKnownState } from "./sync.js";
 
 type Value = JsonValue | AnyRawCoValue;
 
+export { PriorityBasedMessageQueue } from "./PriorityBasedMessageQueue.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
+import {
+  CO_VALUE_LOADING_CONFIG,
+  MAX_RECOMMENDED_TX_SIZE,
+  setCoValueLoadingRetryDelay,
+  setIncomingMessagesTimeBudget,
+} from "./config.js";
 import { LogLevel, logger } from "./logger.js";
-import { getPriorityFromHeader } from "./priority.js";
+import { CO_VALUE_PRIORITY, getPriorityFromHeader } from "./priority.js";
 
 /** @hidden */
 export const cojsonInternals = {
@@ -100,16 +92,16 @@ export const cojsonInternals = {
   accountHeaderForInitialAgentSecret,
   idforHeader,
   StreamingHash,
-  Channel,
   getPriorityFromHeader,
   getGroupDependentKeyList,
   getGroupDependentKey,
   disablePermissionErrors,
   SyncManager,
   CO_VALUE_LOADING_CONFIG,
-  setCoValueLoadingRetryDelay(delay: number) {
-    CO_VALUE_LOADING_CONFIG.RETRY_DELAY = delay;
-  },
+  CO_VALUE_PRIORITY,
+  setIncomingMessagesTimeBudget,
+  setCoValueLoadingRetryDelay,
+  ConnectedPeerChannel,
   textEncoder,
   textDecoder,
 };
@@ -161,10 +153,7 @@ export {
 
 export type {
   Value,
-  IncomingSyncStream,
-  OutgoingSyncQueue,
   DisconnectedError,
-  PingTimeoutError,
   CoValueUniqueness,
   Stringified,
   CoStreamItem,
@@ -174,6 +163,8 @@ export type {
   AccountRole,
 };
 
+export * from "./storage/index.js";
+
 // biome-ignore format: off
 // eslint-disable-next-line @typescript-eslint/no-namespace
 export namespace CojsonInternalTypes {
@@ -196,5 +187,7 @@ export namespace CojsonInternalTypes {
   export type SignerID = import("./crypto/crypto.js").SignerID;
   export type SignerSecret = import("./crypto/crypto.js").SignerSecret;
   export type JsonObject = import("./jsonValue.js").JsonObject;
+  export type OutgoingPeerChannel = import("./sync.js").OutgoingPeerChannel;
+  export type IncomingPeerChannel = import("./sync.js").IncomingPeerChannel;
 }
 // biome-ignore format: on