cojson 0.20.9 → 0.20.11

package/dist/sync.d.ts

@@ -9,7 +9,7 @@ import { CoValuePriority } from "./priority.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { CoValueKnownState, KnownStateSessions } from "./knownState.js";
 import { StorageAPI } from "./storage/index.js";
-export type SyncMessage = LoadMessage | KnownStateMessage | NewContentMessage | DoneMessage;
+export type SyncMessage = LoadMessage | KnownStateMessage | NewContentMessage | DoneMessage | ReconcileMessage | ReconcileAckMessage;
 export type LoadMessage = {
     action: "load";
 } & CoValueKnownState;
@@ -37,6 +37,16 @@ export type DoneMessage = {
     action: "done";
     id: RawCoID;
 };
+export type ReconcileBatchID = string;
+export type ReconcileMessage = {
+    action: "reconcile";
+    id: ReconcileBatchID;
+    values: [coValue: RawCoID, sessionsHash: string][];
+};
+export type ReconcileAckMessage = {
+    action: "reconcile-ack";
+    id: ReconcileBatchID;
+};
 /**
  * Determines when network sync is enabled.
  * - "always": sync is enabled for both Anonymous Authentication and Authenticated Account
@@ -66,14 +76,31 @@ export interface Peer {
     persistent?: boolean;
 }
 export type ServerPeerSelector = (id: RawCoID, serverPeers: PeerState[]) => PeerState[];
+/**
+ * Manages the sync of coValues between peers.
+ * It is responsible for sending, receiving and processing sync messages.
+ * For more details on how the sync protocol works, see the sync protocol documentation:
+ * {@link docs/sync-protocol.md}
+ */
 export declare class SyncManager {
+    #private;
     peers: {
         [key: PeerID]: PeerState;
     };
     local: LocalNode;
+    /**
+     * Tracks pending reconcile acks from the server.
+     */
+    private reconciliationAckTracker;
+    /**
+     * Tracks ongoing storage reconciliation batches in a server.
+     */
+    private ongoingStorageReconciliationTracker;
+    get pendingReconciliationAck(): Map<string, number>;
     private skipVerify;
     private _ignoreUnknownCoValuesFromServers;
     ignoreUnknownCoValuesFromServers(): void;
+    fullStorageReconciliationEnabled: boolean;
     peersCounter: import("@opentelemetry/api").UpDownCounter<import("@opentelemetry/api").Attributes>;
     private transactionsSizeHistogram;
     serverPeerSelector?: ServerPeerSelector;
@@ -86,9 +113,29 @@ export declare class SyncManager {
     getServerPeers(id: RawCoID, excludePeerId?: PeerID): PeerState[];
     getPersistentServerPeers(id: RawCoID): PeerState[];
     handleSyncMessage(msg: SyncMessage, peer: PeerState): void;
-    sendNewContent(id: RawCoID, peer: PeerState, seen?: Set<RawCoID>): void;
+    sendNewContent(id: RawCoID, peer: PeerState, forceKnownReplyOnNoDelta?: boolean): void;
+    /**
+     * Reconciles all in-memory CoValues with all persistent server peers
+     */
     reconcileServerPeers(): void;
+    /**
+     * Ensures all CoValues in storage are synced to the given server peer.
+     * Sends "reconcile" message(s) with [coValueId, sessionsHash] for each CoValue.
+     * Server responds with "known" only where it is missing the CoValue or has different sessions,
+     * so that client can send missing content.
+     * Processes CoValues in batches of RECONCILIATION_BATCH_SIZE.
+     * @param peer - The server peer to reconcile with.
+     * @param initialOffset - Offset to start from (for resuming after interrupt). Default 0.
+     * @param onComplete - Called when reconciliation is fully complete (all batches sent and acked).
+     */
+    startStorageReconciliation(peer: PeerState, initialOffset?: number, onComplete?: () => void): void;
+    private buildStorageReconciliationEntries;
+    private maybeStartStorageReconciliationForPeer;
     resumeUnsyncedCoValues(): Promise<void>;
+    /**
+     * Reconciles all in-memory CoValues with the given peer.
+     * Creates a subscription for each CoValue that is not already subscribed to.
+     */
     startPeerReconciliation(peer: PeerState): void;
     messagesQueue: IncomingMessagesQueue;
     private processing;
@@ -139,9 +186,13 @@ export declare class SyncManager {
      */
     private requestFullContent;
     handleKnownState(msg: KnownStateMessage, peer: PeerState): void;
+    handleReconcile(msg: ReconcileMessage, peer: PeerState): void;
+    handleReconcileAck(msg: ReconcileAckMessage, peer: PeerState): void;
+    private hashKnownStateSessions;
     recordTransactionsSize(newTransactions: Transaction[], source: string): void;
     handleNewContent(msg: NewContentMessage, from: PeerState | "storage" | "import"): void;
     handleCorrection(msg: KnownStateMessage, peer: PeerState): void;
+    private maybeMarkCoValueAsReconciled;
     private syncQueue;
     syncLocalTransaction: (coValue: import("./coValueCore/verifiedState.js").VerifiedState, knownStateBefore: CoValueKnownState) => void;
     trackDirtyCoValues: () => {

package/dist/sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAUvE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAqB,OAAO,EAAE,SAAS,EAAa,MAAM,UAAU,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAGzE,OAAO,EACL,iBAAiB,EAEjB,kBAAkB,EAEnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,iBAAiB,GACjB,iBAAiB,GACjB,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,SAAS,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,eAAe,CAAC;IAC1B,GAAG,EAAE;QACH,CAAC,SAAS,EAAE,SAAS,GAAG,iBAAiB,CAAC;KAC3C,CAAC;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,WAAW,EAAE,CAAC;IAC/B,aAAa,EAAE,SAAS,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,OAAO,CAAC;CACb,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;AAEvD,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAE5B,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAE/C,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,KAAK,IAAI,CAAC;IAC9E,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACrD,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD,MAAM,MAAM,kBAAkB,GAAG,CAC/B,EAAE,EAAE,OAAO,EACX,WAAW,EAAE,SAAS,EAAE,KACrB,SAAS,EAAE,CAAC;AAEjB,qBAAa,WAAW;IACtB,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAM;IACzC,KAAK,EAAE,SAAS,CAAC;IAKjB,OAAO,CAAC,UAAU,CAAkB;IAIpC,OAAO,CAAC,iCAAiC,CAAkB;IAC3D,gCAAgC;IAIhC,YAAY,sFAIT;IACH,OAAO,CAAC,yBAAyB,CAAY;IAE7C,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;gBAE5B,KAAK,EAAE,SAAS;IAc5B,SAAS,EAAE,gBAAgB,CAAC;IAC5B,eAAe,EAAE,uBAAuB,CAAC;IAEzC,8BAA8B;IAI9B,QAAQ,CAAC,EAAE,EAAE,OAAO,GAAG,SAAS,EAAE;IAIlC,cAAc,IAAI,SAAS,EAAE;IAI7B,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE;IAShE,wBAAwB,CAAC,EAAE,EAAE,OAAO,GAAG,SAAS,EAAE;IAIlD,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAuDnD,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,GAAE,GAAG,CAAC,OAAO,CAAa;IAgD3E,oBAAoB;IASd,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IA0E7C,uBAAuB,CAAC,IAAI,EAAE,SAAS;IAuEvC,aAAa,wBAAyD;IACtE,OAAO,CAAC,UAAU,CAAS;IAE3B,WAAW,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAIlD;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;;;;;;;;OASG;YACW,aAAa;IAsD3B,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAkB,GAAE,OAAe;IAuCvD,UAAU,CAAC,MAAM,EAAE,MAAM;IAWzB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW;IAI/C;;;;;;;OAOG;IACH,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IA6D5C;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAcjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAuB/B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAU1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAkB1B,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAoBxD,sBAAsB,CAAC,eAAe,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM;IAUrE,gBAAgB,CACd,GAAG,EAAE,iBAAiB,EACtB,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ;IAsTxC,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAMxD,OAAO,CAAC,SAAS,CAEf;IACF,oBAAoB,iHAAkC;IACtD,kBAAkB;;MAAqC;IAEvD,WAAW,CAAC,OAAO,EAAE,iBAAiB;IA+BtC,OAAO,CAAC,cAAc;IA2CtB,OAAO,CAAC,YAAY;IAiCpB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO;IAQ3C,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IAoDhE,kBAAkB,CAAC,EAAE,EAAE,OAAO;IAI9B,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,SAAS;IAUzC,sBAAsB,CAAC,OAAO,SAAS;IAavC,UAAU,CAAC,OAAO,EAAE,UAAU;IAW9B,aAAa;IAIb;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;CAM9C;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAgCnE"}
+{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AAavE,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAqB,OAAO,EAAE,SAAS,EAAa,MAAM,UAAU,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAChD,OAAO,EAAE,qBAAqB,EAAE,MAAM,kCAAkC,CAAC;AAKzE,OAAO,EACL,iBAAiB,EAEjB,kBAAkB,EAEnB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,UAAU,EAAE,MAAM,oBAAoB,CAAC;AAEhD,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,iBAAiB,GACjB,iBAAiB,GACjB,WAAW,GACX,gBAAgB,GAChB,mBAAmB,CAAC;AAExB,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,SAAS,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,eAAe,CAAC;IAC1B,GAAG,EAAE;QACH,CAAC,SAAS,EAAE,SAAS,GAAG,iBAAiB,CAAC;KAC3C,CAAC;IACF,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAE9B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,WAAW,EAAE,CAAC;IAC/B,aAAa,EAAE,SAAS,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,OAAO,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAEtC,MAAM,MAAM,gBAAgB,GAAG;IAC7B,MAAM,EAAE,WAAW,CAAC;IACpB,EAAE,EAAE,gBAAgB,CAAC;IACrB,MAAM,EAAE,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,CAAC,EAAE,CAAC;CACpD,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG;IAChC,MAAM,EAAE,eAAe,CAAC;IACxB,EAAE,EAAE,gBAAgB,CAAC;CACtB,CAAC;AAEF;;;;;;GAMG;AACH,MAAM,MAAM,QAAQ,GAAG,QAAQ,GAAG,UAAU,GAAG,OAAO,CAAC;AAEvD,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAE5B,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAE/C,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,KAAK,IAAI,CAAC;IAC9E,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACrD,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAMD,MAAM,MAAM,kBAAkB,GAAG,CAC/B,EAAE,EAAE,OAAO,EACX,WAAW,EAAE,SAAS,EAAE,KACrB,SAAS,EAAE,CAAC;AAEjB;;;;;GAKG;AACH,qBAAa,WAAW;;IACtB,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAM;IACzC,KAAK,EAAE,SAAS,CAAC;IACjB;;OAEG;IACH,OAAO,CAAC,wBAAwB,CACc;IAC9C;;OAEG;IACH,OAAO,CAAC,mCAAmC,CACC;IAE5C,IAAI,wBAAwB,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAElD;IAKD,OAAO,CAAC,UAAU,CAAkB;IAIpC,OAAO,CAAC,iCAAiC,CAAkB;IAC3D,gCAAgC;IAIhC,gCAAgC,UAAS;IAEzC,YAAY,sFAIT;IACH,OAAO,CAAC,yBAAyB,CAAY;IAE7C,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;gBAE5B,KAAK,EAAE,SAAS;IAc5B,SAAS,EAAE,gBAAgB,CAAC;IAC5B,eAAe,EAAE,uBAAuB,CAAC;IAEzC,8BAA8B;IAI9B,QAAQ,CAAC,EAAE,EAAE,OAAO,GAAG,SAAS,EAAE;IAIlC,cAAc,IAAI,SAAS,EAAE;IAI7B,cAAc,CAAC,EAAE,EAAE,OAAO,EAAE,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE;IAShE,wBAAwB,CAAC,EAAE,EAAE,OAAO,GAAG,SAAS,EAAE;IAIlD,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAgEnD,cAAc,CACZ,EAAE,EAAE,OAAO,EACX,IAAI,EAAE,SAAS,EACf,wBAAwB,GAAE,OAAe;IA0D3C;;OAEG;IACH,oBAAoB;IASpB;;;;;;;;;OASG;IACH,0BAA0B,CACxB,IAAI,EAAE,SAAS,EACf,aAAa,CAAC,EAAE,MAAM,EACtB,UAAU,CAAC,EAAE,MAAM,IAAI,GACtB,IAAI;IAgFP,OAAO,CAAC,iCAAiC;IAsCzC,OAAO,CAAC,sCAAsC;IAsBxC,sBAAsB,IAAI,OAAO,CAAC,IAAI,CAAC;IA0E7C;;;OAGG;IACH,uBAAuB,CAAC,IAAI,EAAE,SAAS;IA0EvC,aAAa,wBAAyD;IACtE,OAAO,CAAC,UAAU,CAAS;IAE3B,WAAW,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAIlD;;;OAGG;IACH,OAAO,CAAC,wBAAwB;IAQhC;;;;;;;;;OASG;YACW,aAAa;IAsD3B,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,kBAAkB,GAAE,OAAe;IAwCvD,UAAU,CAAC,MAAM,EAAE,MAAM;IAWzB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW;IAI/C;;;;;;;OAOG;IACH,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IA6D5C;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAcjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAuB/B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAU1B;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAkB1B,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IA4BxD,eAAe,CAAC,GAAG,EAAE,gBAAgB,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI;IA2E7D,kBAAkB,CAAC,GAAG,EAAE,mBAAmB,EAAE,IAAI,EAAE,SAAS,GAAG,IAAI;IAWnE,OAAO,CAAC,sBAAsB;IAI9B,sBAAsB,CAAC,eAAe,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM;IAUrE,gBAAgB,CACd,GAAG,EAAE,iBAAiB,EACtB,IAAI,EAAE,SAAS,GAAG,SAAS,GAAG,QAAQ;IAyTxC,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAMxD,OAAO,CAAC,4BAA4B;IAcpC,OAAO,CAAC,SAAS,CAEf;IACF,oBAAoB,iHAAkC;IACtD,kBAAkB;;MAAqC;IAEvD,WAAW,CAAC,OAAO,EAAE,iBAAiB;IA+BtC,OAAO,CAAC,cAAc;IA2CtB,OAAO,CAAC,YAAY;IAiCpB;;;;OAIG;IACH,qBAAqB,CAAC,EAAE,EAAE,OAAO,GAAG,OAAO;IAQ3C,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IAoDhE,kBAAkB,CAAC,EAAE,EAAE,OAAO;IAI9B,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,SAAS;IAUzC,sBAAsB,CAAC,OAAO,SAAS;IAavC,UAAU,CAAC,OAAO,EAAE,UAAU;IAW9B,aAAa;IAIb;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS;CAM9C;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,MAAM,GAAG,kBAAkB,CAgCnE"}

package/dist/sync.js

@@ -1,24 +1,51 @@
+var __classPrivateFieldGet = (this && this.__classPrivateFieldGet) || function (receiver, state, kind, f) {
+    if (kind === "a" && !f) throw new TypeError("Private accessor was defined without a getter");
+    if (typeof state === "function" ? receiver !== state || !f : !state.has(receiver)) throw new TypeError("Cannot read private member from an object whose class did not declare it");
+    return kind === "m" ? f : kind === "a" ? f.call(receiver) : f ? f.value : state.get(receiver);
+};
+var _SyncManager_instances, _SyncManager_sendNewContent;
+import { base58 } from "@scure/base";
 import { md5 } from "@noble/hashes/legacy";
 import { ValueType, metrics } from "@opentelemetry/api";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { UnsyncedCoValuesTracker } from "./UnsyncedCoValuesTracker.js";
-import { SYNC_SCHEDULER_CONFIG } from "./config.js";
+import { STORAGE_RECONCILIATION_CONFIG, SYNC_SCHEDULER_CONFIG, } from "./config.js";
 import { getContenDebugInfo, getNewTransactionsFromContentMessage, getSessionEntriesFromContentMessage, getTransactionSize, knownStateFromContent, } from "./coValueContentMessage.js";
 import { isDeleteSessionID, isRawCoID } from "./ids.js";
 import { logger } from "./logger.js";
 import { IncomingMessagesQueue } from "./queue/IncomingMessagesQueue.js";
 import { LocalTransactionsSyncQueue } from "./queue/LocalTransactionsSyncQueue.js";
+import { OngoingStorageReconciliationTracker } from "./OngoingStorageReconciliationTracker.js";
+import { StorageReconciliationServerAckTracker } from "./StorageReconciliationAckTracker.js";
 import { knownStateFrom, peerHasAllContent, } from "./knownState.js";
 function isPersistentServerPeer(peer) {
     return peer.role === "server" && (peer.persistent ?? false);
 }
+/**
+ * Manages the sync of coValues between peers.
+ * It is responsible for sending, receiving and processing sync messages.
+ * For more details on how the sync protocol works, see the sync protocol documentation:
+ * {@link docs/sync-protocol.md}
+ */
 export class SyncManager {
+    get pendingReconciliationAck() {
+        return this.reconciliationAckTracker.pendingReconciliationAck;
+    }
     ignoreUnknownCoValuesFromServers() {
         this._ignoreUnknownCoValuesFromServers = true;
     }
     constructor(local) {
+        _SyncManager_instances.add(this);
         this.peers = {};
+        /**
+         * Tracks pending reconcile acks from the server.
+         */
+        this.reconciliationAckTracker = new StorageReconciliationServerAckTracker();
+        /**
+         * Tracks ongoing storage reconciliation batches in a server.
+         */
+        this.ongoingStorageReconciliationTracker = new OngoingStorageReconciliationTracker();
         // When true, transactions will not be verified.
         // This is useful when syncing only for storage purposes, with the expectation that
         // the transactions have already been verified by the [trusted] peer that sent them.
@@ -26,6 +53,7 @@ export class SyncManager {
         // When true, coValues that arrive from server peers will be ignored if they had not
         // explicitly been requested via a load message.
         this._ignoreUnknownCoValuesFromServers = false;
+        this.fullStorageReconciliationEnabled = false;
         this.peersCounter = metrics.getMeter("cojson").createUpDownCounter("jazz.peers", {
             description: "Amount of connected peers",
             valueType: ValueType.INT,
@@ -66,6 +94,14 @@ export class SyncManager {
         return this.getServerPeers(id).filter((peer) => peer.persistent);
     }
     handleSyncMessage(msg, peer) {
+        if (msg.action === "reconcile") {
+            this.handleReconcile(msg, peer);
+            return;
+        }
+        if (msg.action === "reconcile-ack") {
+            this.handleReconcileAck(msg, peer);
+            return;
+        }
         if (!isRawCoID(msg.id)) {
             const errorType = msg.id ? "invalid" : "undefined";
             logger.warn(`Received sync message with ${errorType} id`, {
@@ -109,48 +145,136 @@ export class SyncManager {
                 throw new Error(`Unknown message type ${msg.action}`);
         }
     }
-    sendNewContent(id, peer, seen = new Set()) {
-        if (seen.has(id)) {
-            return;
+    sendNewContent(id, peer, forceKnownReplyOnNoDelta = false) {
+        __classPrivateFieldGet(this, _SyncManager_instances, "m", _SyncManager_sendNewContent).call(this, id, peer, new Set(), forceKnownReplyOnNoDelta);
+    }
+    /**
+     * Reconciles all in-memory CoValues with all persistent server peers
+     */
+    reconcileServerPeers() {
+        const serverPeers = Object.values(this.peers).filter(isPersistentServerPeer);
+        for (const peer of serverPeers) {
+            this.startPeerReconciliation(peer);
         }
-        seen.add(id);
-        const coValue = this.local.getCoValue(id);
-        if (!coValue.isAvailable()) {
+    }
+    /**
+     * Ensures all CoValues in storage are synced to the given server peer.
+     * Sends "reconcile" message(s) with [coValueId, sessionsHash] for each CoValue.
+     * Server responds with "known" only where it is missing the CoValue or has different sessions,
+     * so that client can send missing content.
+     * Processes CoValues in batches of RECONCILIATION_BATCH_SIZE.
+     * @param peer - The server peer to reconcile with.
+     * @param initialOffset - Offset to start from (for resuming after interrupt). Default 0.
+     * @param onComplete - Called when reconciliation is fully complete (all batches sent and acked).
+     */
+    startStorageReconciliation(peer, initialOffset, onComplete) {
+        if (!this.local.storage)
             return;
-        }
-        const includeDependencies = peer.role !== "server";
-        if (includeDependencies) {
-            for (const dependency of coValue.getDependedOnCoValues()) {
-                this.sendNewContent(dependency, peer, seen);
-            }
-        }
-        const newContentPieces = coValue.newContentSince(peer.getOptimisticKnownState(id));
-        if (newContentPieces) {
-            for (const piece of newContentPieces) {
-                this.trySendToPeer(peer, piece);
-            }
-            peer.combineOptimisticWith(id, coValue.knownState());
-        }
-        else if (!peer.toldKnownState.has(id)) {
-            if (coValue.isDeleted) {
-                // This way we make the peer believe that we've always ingested all the content they sent, even though we skipped it because the coValue is deleted
-                this.trySendToPeer(peer, coValue.stopSyncingKnownStateMessage(peer.getKnownState(id)));
-            }
-            else {
+        if (!isPersistentServerPeer(peer))
+            return;
+        const startOffset = initialOffset ?? 0;
+        const batchSize = STORAGE_RECONCILIATION_CONFIG.BATCH_SIZE;
+        const storage = this.local.storage;
+        storage.getCoValueCount((totalCoValueCount) => {
+            const sendReconcileMessage = (batchId, entries, offset) => {
+                if (entries.length === 0)
+                    return;
+                this.reconciliationAckTracker.trackBatch(batchId, peer.id, offset + batchSize);
                 this.trySendToPeer(peer, {
-                    action: "known",
-                    ...coValue.knownStateWithStreaming(),
+                    action: "reconcile",
+                    id: batchId,
+                    values: entries,
                 });
-            }
-        }
-        peer.trackToldKnownState(id);
+            };
+            const triggerNextBatch = (lastBatchLength, offset) => {
+                // This value becomes false when the last covalueid batch picked from the storage
+                // is smaller than the batch size.
+                if (lastBatchLength === batchSize) {
+                    logger.info("Reconciled CoValues in storage", {
+                        peerId: peer.id,
+                        completed: offset + batchSize,
+                        total: totalCoValueCount,
+                    });
+                    processStorageBatch(offset + batchSize);
+                }
+                else {
+                    // Note: `completed` can be higher than `total` if CoValues were added
+                    // after the reconciliation started
+                    logger.info("Storage reconciliation complete", {
+                        peerId: peer.id,
+                        startOffset,
+                        completed: offset + lastBatchLength,
+                        total: totalCoValueCount,
+                    });
+                    onComplete?.();
+                }
+            };
+            const processStorageBatch = (offset) => {
+                storage.getCoValueIDs(batchSize, offset, (batch) => {
+                    this.buildStorageReconciliationEntries(batch, (entries) => {
+                        if (entries.length === 0) {
+                            triggerNextBatch(batch.length, offset);
+                            return;
+                        }
+                        const batchId = base58.encode(this.local.crypto.randomBytes(12));
+                        sendReconcileMessage(batchId, entries, offset);
+                        this.reconciliationAckTracker.waitForAck(batchId, peer, () => {
+                            triggerNextBatch(batch.length, offset);
+                        });
+                    });
+                });
+            };
+            logger.info("Starting storage reconciliation", {
+                peerId: peer.id,
+                startOffset,
+                total: totalCoValueCount,
+            });
+            processStorageBatch(startOffset);
+        });
     }
-    reconcileServerPeers() {
-        const serverPeers = Object.values(this.peers).filter((peer) => peer.role === "server");
-        for (const peer of serverPeers) {
-            this.startPeerReconciliation(peer);
+    buildStorageReconciliationEntries(batch, callback) {
+        const storage = this.local.storage;
+        if (!storage) {
+            callback([]);
+            return;
+        }
+        const pending = batch.filter(({ id }) => !this.local.isCoValueInMemory(id));
+        if (pending.length === 0) {
+            callback([]);
+            return;
+        }
+        let done = 0;
+        const entries = [];
+        for (const coValue of pending) {
+            storage.loadKnownState(coValue.id, (storageKnownState) => {
+                if (storageKnownState) {
+                    entries.push([
+                        coValue.id,
+                        this.hashKnownStateSessions(storageKnownState.sessions),
+                    ]);
+                }
+                done += 1;
+                if (done === pending.length) {
+                    callback(entries);
+                }
+            });
         }
     }
+    maybeStartStorageReconciliationForPeer(peer) {
+        if (!this.fullStorageReconciliationEnabled)
+            return;
+        if (!this.local.storage)
+            return;
+        const sessionId = this.local.currentSessionID;
+        this.local.storage.tryAcquireStorageReconciliationLock(sessionId, peer.id, (result) => {
+            if (!result.acquired)
+                return;
+            const lastProcessedOffset = result.lastProcessedOffset;
+            this.startStorageReconciliation(peer, lastProcessedOffset, () => {
+                this.local.storage?.releaseStorageReconciliationLock(sessionId, peer.id);
+            });
+        });
+    }
     async resumeUnsyncedCoValues() {
         if (!this.local.storage) {
             // No storage available, skip resumption
@@ -207,12 +331,18 @@ export class SyncManager {
             });
         });
     }
+    /**
+     * Reconciles all in-memory CoValues with the given peer.
+     * Creates a subscription for each CoValue that is not already subscribed to.
+     */
     startPeerReconciliation(peer) {
         if (isPersistentServerPeer(peer)) {
             // Resume syncing unsynced CoValues asynchronously
             this.resumeUnsyncedCoValues().catch((error) => {
                 logger.warn("Failed to resume unsynced CoValues:", error);
             });
+            // Try to run full storage reconciliation for this peer (scheduled per peer, every 30 days)
+            this.maybeStartStorageReconciliationForPeer(peer);
         }
         const coValuesOrderedByDependency = [];
         const seen = new Set();
@@ -360,6 +490,7 @@ export class SyncManager {
         });
         peerState.addCloseListener(() => {
             unsubscribeFromKnownStatesUpdates();
+            this.ongoingStorageReconciliationTracker.clearPeer(peer.id);
             this.peersCounter.add(-1, { role: peer.role });
             if (!peer.persistent && this.peers[peer.id] === peerState) {
                 this.removePeer(peer.id);
@@ -398,7 +529,7 @@ export class SyncManager {
         const coValue = this.local.getCoValue(msg.id);
         // Fast path: CoValue is already in memory
         if (coValue.isAvailable()) {
-            this.sendNewContent(msg.id, peer);
+            this.sendNewContent(msg.id, peer, true);
             return;
         }
         const peerKnownState = peer.getOptimisticKnownState(msg.id);
@@ -411,7 +542,7 @@ export class SyncManager {
         coValue.getKnownStateFromStorage((storageKnownState) => {
             // Race condition: CoValue might have been loaded while we were waiting for storage
             if (coValue.isAvailable()) {
-                this.sendNewContent(msg.id, peer);
+                this.sendNewContent(msg.id, peer, true);
                 return;
             }
             if (!storageKnownState) {
@@ -431,7 +562,7 @@ export class SyncManager {
                 // Even though we responded with KNOWN (client has everything), we need
                 // to establish a subscription so that updates from core flow to us.
                 const serverPeers = this.getServerPeers(msg.id, peer.id);
-                coValue.loadFromPeers(serverPeers);
+                coValue.loadFromPeers(serverPeers, "low-priority");
                 return;
             }
             // Peer needs content - do full load from storage
@@ -445,7 +576,7 @@ export class SyncManager {
     loadFromStorageAndRespond(id, peer, coValue) {
         coValue.loadFromStorage((found) => {
             if (found && coValue.isAvailable()) {
-                this.sendNewContent(id, peer);
+                this.sendNewContent(id, peer, true);
             }
             else {
                 this.loadFromPeersAndRespond(id, peer, coValue);
@@ -457,7 +588,7 @@ export class SyncManager {
      */
     loadFromPeersAndRespond(id, peer, coValue) {
         const peers = this.getServerPeers(id, peer.id);
-        coValue.loadFromPeers(peers);
+        coValue.loadFromPeers(peers, "immediate");
         const handleLoadResult = () => {
             if (coValue.isAvailable()) {
                 this.sendNewContent(id, peer);
@@ -517,7 +648,84 @@ export class SyncManager {
         if (coValue.isAvailable()) {
             this.sendNewContent(msg.id, peer);
         }
-        peer.trackLoadRequestComplete(coValue);
+        else if (coValue.isKnownStateAvailable()) {
+            // Validate if content is missing before loading it from storage
+            if (!this.syncState.isSynced(peer, msg.id)) {
+                this.local.loadCoValueCore(msg.id).then(() => {
+                    this.sendNewContent(msg.id, peer);
+                });
+            }
+        }
+        peer.trackLoadRequestComplete(coValue, "known");
+        this.maybeMarkCoValueAsReconciled(peer, msg.id);
+    }
+    handleReconcile(msg, peer) {
+        let remaining = msg.values.length;
+        if (remaining === 0) {
+            this.trySendToPeer(peer, { action: "reconcile-ack", id: msg.id });
+            return;
+        }
+        const pending = new Set();
+        const processEntryDone = () => {
+            remaining -= 1;
+            if (remaining !== 0) {
+                return;
+            }
+            if (pending.size === 0) {
+                this.trySendToPeer(peer, { action: "reconcile-ack", id: msg.id });
+                return;
+            }
+            this.ongoingStorageReconciliationTracker.trackBatch(peer.id, msg.id, pending);
+        };
+        for (const [coValueId, clientSessionsHash] of msg.values) {
+            // Avoid creating a new coValue object if it's not already in memory
+            const inMemoryCoValue = this.local.isCoValueInMemory(coValueId)
+                ? this.local.getCoValue(coValueId)
+                : undefined;
+            if (inMemoryCoValue?.isErroredInPeer(peer.id)) {
+                processEntryDone();
+                continue;
+            }
+            const maybeSendLoadRequest = (knownState) => {
+                if (!knownState) {
+                    pending.add(coValueId);
+                    peer.trackToldKnownState(coValueId);
+                    this.trySendToPeer(peer, {
+                        action: "load",
+                        id: coValueId,
+                        header: false,
+                        sessions: {},
+                    });
+                }
+                else {
+                    const serverSessionsHash = this.hashKnownStateSessions(knownState.sessions);
+                    if (serverSessionsHash !== clientSessionsHash) {
+                        pending.add(coValueId);
+                        peer.trackToldKnownState(coValueId);
+                        this.trySendToPeer(peer, { action: "load", ...knownState });
+                    }
+                }
+                processEntryDone();
+            };
+            if (inMemoryCoValue?.isAvailable() ||
+                inMemoryCoValue?.loadingState === "onlyKnownState") {
+                maybeSendLoadRequest(inMemoryCoValue.knownState());
+            }
+            else {
+                this.local.storage
+                    ? this.local.storage.loadKnownState(coValueId, maybeSendLoadRequest)
+                    : maybeSendLoadRequest(undefined);
+            }
+        }
+    }
+    handleReconcileAck(msg, peer) {
+        const nextOffset = this.reconciliationAckTracker.handleAck(msg.id, peer.id);
+        if (nextOffset !== undefined) {
+            this.local.storage?.renewStorageReconciliationLock(this.local.currentSessionID, peer.id, nextOffset);
+        }
+    }
+    hashKnownStateSessions(sessions) {
+        return this.local.crypto.shortHash(sessions);
     }
     recordTransactionsSize(newTransactions, source) {
         for (const tx of newTransactions) {
@@ -759,7 +967,10 @@ export class SyncManager {
                 this.trackSyncState(coValue.id);
             }
         }
-        peer?.trackLoadRequestComplete(coValue);
+        peer?.trackLoadRequestComplete(coValue, "content");
+        if (peer && !coValue.isStreaming()) {
+            this.maybeMarkCoValueAsReconciled(peer, msg.id);
+        }
         for (const peer of this.getPeers(coValue.id)) {
             /**
              * We sync the content against the source peer if it is a client or server peers
@@ -774,7 +985,7 @@ export class SyncManager {
                 this.sendNewContent(coValue.id, peer);
             }
             else if (peer.role === "server") {
-                peer.sendLoadRequest(coValue);
+                peer.sendLoadRequest(coValue, "low-priority");
             }
         }
     }
@@ -782,6 +993,15 @@ export class SyncManager {
         peer.setKnownState(msg.id, knownStateFrom(msg));
         return this.sendNewContent(msg.id, peer);
     }
+    maybeMarkCoValueAsReconciled(peer, coValueId) {
+        const completedBatchIds = this.ongoingStorageReconciliationTracker.markItemComplete(peer.id, coValueId);
+        for (const batchId of completedBatchIds) {
+            this.trySendToPeer(peer, {
+                action: "reconcile-ack",
+                id: batchId,
+            });
+        }
+    }
     syncContent(content) {
         const coValue = this.local.getCoValue(content.id);
         this.storeContent(content);
@@ -949,6 +1169,42 @@ export class SyncManager {
         return this.unsyncedTracker.forcePersist();
     }
 }
+_SyncManager_instances = new WeakSet(), _SyncManager_sendNewContent = function _SyncManager_sendNewContent(id, peer, seen, forceKnownReplyOnNoDelta) {
+    if (seen.has(id)) {
+        return;
+    }
+    seen.add(id);
+    const coValue = this.local.getCoValue(id);
+    if (!coValue.isAvailable()) {
+        return;
+    }
+    const includeDependencies = peer.role !== "server";
+    if (includeDependencies) {
+        for (const dependency of coValue.getDependedOnCoValues()) {
+            __classPrivateFieldGet(this, _SyncManager_instances, "m", _SyncManager_sendNewContent).call(this, dependency, peer, seen, false);
+        }
+    }
+    const newContentPieces = coValue.newContentSince(peer.getOptimisticKnownState(id));
+    if (newContentPieces) {
+        for (const piece of newContentPieces) {
+            this.trySendToPeer(peer, piece);
+        }
+        peer.combineOptimisticWith(id, coValue.knownState());
+    }
+    else if (forceKnownReplyOnNoDelta || !peer.toldKnownState.has(id)) {
+        if (coValue.isDeleted) {
+            // This way we make the peer believe that we've always ingested all the content they sent, even though we skipped it because the coValue is deleted
+            this.trySendToPeer(peer, coValue.stopSyncingKnownStateMessage(peer.getKnownState(id)));
+        }
+        else {
+            this.trySendToPeer(peer, {
+                action: "known",
+                ...coValue.knownStateWithStreaming(),
+            });
+        }
+    }
+    peer.trackToldKnownState(id);
+};
 /**
  * Returns a ServerPeerSelector that implements the Highest Weighted Random (HWR) algorithm.
  *