cojson 0.15.8 → 0.15.9

package/dist/storage/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/storage/types.ts"],"names":[],"mappings":""}

package/dist/streamUtils.d.ts

@@ -1,13 +1,17 @@
-import { Channel } from "queueueue";
-import { Peer, PeerID, SyncMessage } from "./sync.js";
-export { Channel } from "queueueue";
-export declare function connectedPeers(peer1id: PeerID, peer2id: PeerID, { peer1role, peer2role, crashOnClose, }?: {
+import { DisconnectedError, IncomingPeerChannel, OutgoingPeerChannel, Peer, PeerID, SyncMessage } from "./sync.js";
+export declare function connectedPeers(peer1id: PeerID, peer2id: PeerID, { peer1role, peer2role, }?: {
     peer1role?: Peer["role"];
     peer2role?: Peer["role"];
-    crashOnClose?: boolean;
 }): [Peer, Peer];
-export declare function newQueuePair(): [
-    AsyncIterable<SyncMessage>,
-    Channel<SyncMessage>
-];
+export declare function newQueuePair(): [ConnectedPeerChannel, ConnectedPeerChannel];
+export declare class ConnectedPeerChannel implements IncomingPeerChannel, OutgoingPeerChannel {
+    buffer: (SyncMessage | DisconnectedError)[];
+    push(msg: SyncMessage | DisconnectedError): void;
+    close(): void;
+    listeners: Set<(msg: SyncMessage | DisconnectedError) => void>;
+    onMessage(callback: (msg: SyncMessage | DisconnectedError) => void): void;
+    closed: boolean;
+    closeListeners: Set<() => void>;
+    onClose(callback: () => void): void;
+}
 //# sourceMappingURL=streamUtils.d.ts.map

package/dist/streamUtils.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"streamUtils.d.ts","sourceRoot":"","sources":["../src/streamUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACtD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,EACE,SAAoB,EACpB,SAAoB,EACpB,YAAoB,GACrB,GAAE;IACD,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,YAAY,CAAC,EAAE,OAAO,CAAC;CACnB,GACL,CAAC,IAAI,EAAE,IAAI,CAAC,CAqBd;AAED,wBAAgB,YAAY,IAAI;IAC9B,aAAa,CAAC,WAAW,CAAC;IAC1B,OAAO,CAAC,WAAW,CAAC;CACrB,CAIA"}
+{"version":3,"file":"streamUtils.d.ts","sourceRoot":"","sources":["../src/streamUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,mBAAmB,EACnB,mBAAmB,EACnB,IAAI,EACJ,MAAM,EACN,WAAW,EACZ,MAAM,WAAW,CAAC;AAEnB,wBAAgB,cAAc,CAC5B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,MAAM,EACf,EACE,SAAoB,EACpB,SAAoB,GACrB,GAAE;IACD,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;CACrB,GACL,CAAC,IAAI,EAAE,IAAI,CAAC,CAmBd;AAED,wBAAgB,YAAY,IAAI,CAAC,oBAAoB,EAAE,oBAAoB,CAAC,CAI3E;AAED,qBAAa,oBACX,YAAW,mBAAmB,EAAE,mBAAmB;IAEnD,MAAM,EAAE,CAAC,WAAW,GAAG,iBAAiB,CAAC,EAAE,CAAM;IAEjD,IAAI,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB;IAWzC,KAAK;IASL,SAAS,YAAiB,WAAW,GAAG,iBAAiB,KAAK,IAAI,EAAI;IACtE,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI;IAWlE,MAAM,UAAS;IACf,cAAc,YAAiB,IAAI,EAAI;IACvC,OAAO,CAAC,QAAQ,EAAE,MAAM,IAAI;CAG7B"}

package/dist/streamUtils.js

@@ -1,26 +1,59 @@
-import { Channel } from "queueueue";
-export { Channel } from "queueueue";
-export function connectedPeers(peer1id, peer2id, { peer1role = "client", peer2role = "client", crashOnClose = false, } = {}) {
-    const [from1to2Rx, from1to2Tx] = newQueuePair();
-    const [from2to1Rx, from2to1Tx] = newQueuePair();
+export function connectedPeers(peer1id, peer2id, { peer1role = "client", peer2role = "client", } = {}) {
+    const from1to2 = new ConnectedPeerChannel();
+    const from2to1 = new ConnectedPeerChannel();
     const peer2AsPeer = {
         id: peer2id,
-        incoming: from2to1Rx,
-        outgoing: from1to2Tx,
+        incoming: from2to1,
+        outgoing: from1to2,
         role: peer2role,
-        crashOnClose: crashOnClose,
     };
     const peer1AsPeer = {
         id: peer1id,
-        incoming: from1to2Rx,
-        outgoing: from2to1Tx,
+        incoming: from1to2,
+        outgoing: from2to1,
         role: peer1role,
-        crashOnClose: crashOnClose,
     };
     return [peer1AsPeer, peer2AsPeer];
 }
 export function newQueuePair() {
-    const channel = new Channel();
-    return [channel.wrap(), channel];
+    const channel = new ConnectedPeerChannel();
+    return [channel, channel];
+}
+export class ConnectedPeerChannel {
+    constructor() {
+        this.buffer = [];
+        this.listeners = new Set();
+        this.closed = false;
+        this.closeListeners = new Set();
+    }
+    push(msg) {
+        if (!this.listeners.size) {
+            this.buffer.push(msg);
+            return;
+        }
+        for (const listener of this.listeners) {
+            listener(msg);
+        }
+    }
+    close() {
+        this.closed = true;
+        for (const listener of this.closeListeners) {
+            listener();
+        }
+        this.closeListeners.clear();
+        this.listeners.clear();
+    }
+    onMessage(callback) {
+        if (this.buffer.length) {
+            for (const msg of this.buffer) {
+                callback(msg);
+            }
+            this.buffer = [];
+        }
+        this.listeners.add(callback);
+    }
+    onClose(callback) {
+        this.closeListeners.add(callback);
+    }
 }
 //# sourceMappingURL=streamUtils.js.map

package/dist/streamUtils.js.map

@@ -1 +1 @@
-{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../src/streamUtils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAEpC,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,OAAe,EACf,EACE,SAAS,GAAG,QAAQ,EACpB,SAAS,GAAG,QAAQ,EACpB,YAAY,GAAG,KAAK,MAKlB,EAAE;IAEN,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG,YAAY,EAAE,CAAC;IAChD,MAAM,CAAC,UAAU,EAAE,UAAU,CAAC,GAAG,YAAY,EAAE,CAAC;IAEhD,MAAM,WAAW,GAAS;QACxB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,YAAY;KAC3B,CAAC;IAEF,MAAM,WAAW,GAAS;QACxB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,UAAU;QACpB,QAAQ,EAAE,UAAU;QACpB,IAAI,EAAE,SAAS;QACf,YAAY,EAAE,YAAY;KAC3B,CAAC;IAEF,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,YAAY;IAI1B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAe,CAAC;IAE3C,OAAO,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC;AACnC,CAAC"}
+{"version":3,"file":"streamUtils.js","sourceRoot":"","sources":["../src/streamUtils.ts"],"names":[],"mappings":"AASA,MAAM,UAAU,cAAc,CAC5B,OAAe,EACf,OAAe,EACf,EACE,SAAS,GAAG,QAAQ,EACpB,SAAS,GAAG,QAAQ,MAIlB,EAAE;IAEN,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC5C,MAAM,QAAQ,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE5C,MAAM,WAAW,GAAS;QACxB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,SAAS;KAChB,CAAC;IAEF,MAAM,WAAW,GAAS;QACxB,EAAE,EAAE,OAAO;QACX,QAAQ,EAAE,QAAQ;QAClB,QAAQ,EAAE,QAAQ;QAClB,IAAI,EAAE,SAAS;KAChB,CAAC;IAEF,OAAO,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;AACpC,CAAC;AAED,MAAM,UAAU,YAAY;IAC1B,MAAM,OAAO,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAE3C,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;AAC5B,CAAC;AAED,MAAM,OAAO,oBAAoB;IAAjC;QAGE,WAAM,GAAwC,EAAE,CAAC;QAsBjD,cAAS,GAAG,IAAI,GAAG,EAAkD,CAAC;QAYtE,WAAM,GAAG,KAAK,CAAC;QACf,mBAAc,GAAG,IAAI,GAAG,EAAc,CAAC;IAIzC,CAAC;IArCC,IAAI,CAAC,GAAoC;QACvC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtB,OAAO;QACT,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACtC,QAAQ,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IAED,KAAK;QACH,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,KAAK,MAAM,QAAQ,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YAC3C,QAAQ,EAAE,CAAC;QACb,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACzB,CAAC;IAGD,SAAS,CAAC,QAAwD;QAChE,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC9B,QAAQ,CAAC,GAAG,CAAC,CAAC;YAChB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,EAAE,CAAC;QACnB,CAAC;QAED,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC/B,CAAC;IAID,OAAO,CAAC,QAAoB;QAC1B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACpC,CAAC;CACF"}

package/dist/sync.d.ts

@@ -1,3 +1,4 @@
+import { IncomingMessagesQueue } from "./IncomingMessagesQueue.js";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { CoValueCore } from "./coValueCore/coValueCore.js";
@@ -31,6 +32,9 @@ export type NewContentMessage = {
     new: {
         [sessionID: SessionID]: SessionNewContent;
     };
+    expectContentUntil?: {
+        [sessionID: SessionID]: number;
+    };
 };
 export type SessionNewContent = {
     after: number;
@@ -43,19 +47,22 @@ export type DoneMessage = {
 };
 export type PeerID = string;
 export type DisconnectedError = "Disconnected";
-export type PingTimeoutError = "PingTimeout";
-export type IncomingSyncStream = AsyncIterable<SyncMessage | DisconnectedError | PingTimeoutError>;
-export type OutgoingSyncQueue = {
-    push: (msg: SyncMessage) => Promise<unknown>;
+export interface IncomingPeerChannel {
     close: () => void;
-};
+    onMessage: (callback: (msg: SyncMessage | DisconnectedError) => void) => void;
+    onClose: (callback: () => void) => void;
+}
+export interface OutgoingPeerChannel {
+    push: (msg: SyncMessage | DisconnectedError) => void;
+    close: () => void;
+    onClose: (callback: () => void) => void;
+}
 export interface Peer {
     id: PeerID;
-    incoming: IncomingSyncStream;
-    outgoing: OutgoingSyncQueue;
-    role: "server" | "client" | "storage";
+    incoming: IncomingPeerChannel;
+    outgoing: OutgoingPeerChannel;
+    role: "server" | "client";
     priority?: number;
-    crashOnClose: boolean;
     deletePeerStateOnClose?: boolean;
 }
 export declare function combinedKnownStates(stateA: CoValueKnownState, stateB: CoValueKnownState): CoValueKnownState;
@@ -70,11 +77,12 @@ export declare class SyncManager {
     syncState: SyncStateManager;
     peersInPriorityOrder(): PeerState[];
     getPeers(): PeerState[];
-    getServerAndStoragePeers(excludePeerId?: PeerID): PeerState[];
-    hasStoragePeers(): boolean;
+    getServerPeers(excludePeerId?: PeerID): PeerState[];
     handleSyncMessage(msg: SyncMessage, peer: PeerState): void;
     sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState, seen?: Set<RawCoID>): void;
     startPeerReconciliation(peer: PeerState): void;
+    messagesQueue: IncomingMessagesQueue;
+    pushMessage(incoming: SyncMessage, peer: PeerState): void;
     addPeer(peer: Peer): void;
     trySendToPeer(peer: PeerState, msg: SyncMessage): void;
     /**
@@ -88,14 +96,14 @@ export declare class SyncManager {
     handleLoad(msg: LoadMessage, peer: PeerState): void;
     handleKnownState(msg: KnownStateMessage, peer: PeerState): void;
     recordTransactionsSize(newTransactions: Transaction[], source: string): void;
-    handleNewContent(msg: NewContentMessage, peer: PeerState): void;
+    handleNewContent(msg: NewContentMessage, from: PeerState | "storage"): void;
     handleCorrection(msg: KnownStateMessage, peer: PeerState): void;
-    handleUnsubscribe(_msg: DoneMessage): void;
     requestedSyncs: Set<`co_z${string}`>;
     requestCoValueSync(coValue: CoValueCore): void;
+    storeCoValue(coValue: CoValueCore, data: NewContentMessage[] | undefined): void;
     syncCoValue(coValue: CoValueCore): void;
     waitForSyncWithPeer(peerId: PeerID, id: RawCoID, timeout: number): Promise<unknown> | undefined;
-    waitForStorageSync(id: RawCoID, timeout?: number): Promise<unknown[]>;
+    waitForStorageSync(id: RawCoID): Promise<void> | undefined;
     waitForSync(id: RawCoID, timeout?: number): Promise<unknown[]>;
     waitForAllCoValuesSync(timeout?: number): Promise<unknown[][]>;
     gracefulShutdown(): void;

package/dist/sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAEL,WAAW,EACZ,MAAM,8BAA8B,CAAC;AAEtC,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIhD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE;QAAE,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAAA;KAAE,CAAC;CAC9C,CAAC;AAEF,wBAAgB,eAAe,CAAC,EAAE,EAAE,OAAO,GAAG,iBAAiB,CAM9D;AAED,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,iBAAiB,GACjB,iBAAiB,GACjB,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,SAAS,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,eAAe,CAAC;IAC1B,GAAG,EAAE;QACH,CAAC,SAAS,EAAE,SAAS,GAAG,iBAAiB,CAAC;KAC3C,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,WAAW,EAAE,CAAC;IAC/B,aAAa,EAAE,SAAS,CAAC;CAC1B,CAAC;AACF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,OAAO,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAE5B,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAE/C,MAAM,MAAM,gBAAgB,GAAG,aAAa,CAAC;AAE7C,MAAM,MAAM,kBAAkB,GAAG,aAAa,CAC5C,WAAW,GAAG,iBAAiB,GAAG,gBAAgB,CACnD,CAAC;AACF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,KAAK,OAAO,CAAC,OAAO,CAAC,CAAC;IAC7C,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB,CAAC;AAEF,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,IAAI,EAAE,QAAQ,GAAG,QAAQ,GAAG,SAAS,CAAC;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,OAAO,CAAC;IACtB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,iBAAiB,EACzB,MAAM,EAAE,iBAAiB,GACxB,iBAAiB,CAoBnB;AAED,qBAAa,WAAW;IACtB,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAM;IACzC,KAAK,EAAE,SAAS,CAAC;IAEjB,YAAY,sFAIT;IACH,OAAO,CAAC,yBAAyB,CAAY;gBAEjC,KAAK,EAAE,SAAS;IAa5B,SAAS,EAAE,gBAAgB,CAAC;IAE5B,oBAAoB,IAAI,SAAS,EAAE;IASnC,QAAQ,IAAI,SAAS,EAAE;IAIvB,wBAAwB,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE;IAS7D,eAAe,IAAI,OAAO;IAM1B,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAuCnD,mCAAmC,CACjC,EAAE,EAAE,OAAO,EACX,IAAI,EAAE,SAAS,EACf,IAAI,GAAE,GAAG,CAAC,OAAO,CAAa;IAsChC,uBAAuB,CAAC,IAAI,EAAE,SAAS;IA+DvC,OAAO,CAAC,IAAI,EAAE,IAAI;IA2DlB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW;IAI/C;;;;;;;OAOG;IACH,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IA+D5C,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAkBxD,sBAAsB,CAAC,eAAe,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM;IAarE,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAiOxD,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAMxD,iBAAiB,CAAC,IAAI,EAAE,WAAW;IAEnC,cAAc,uBAAsB;IACpC,kBAAkB,CAAC,OAAO,EAAE,WAAW;IAcvC,WAAW,CAAC,OAAO,EAAE,WAAW;IAuBhC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IA4ChE,kBAAkB,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,SAAS;IAUhD,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,SAAS;IAQzC,sBAAsB,CAAC,OAAO,SAAS;IAavC,gBAAgB;CAKjB"}
+{"version":3,"file":"sync.d.ts","sourceRoot":"","sources":["../src/sync.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,qBAAqB,EAAE,MAAM,4BAA4B,CAAC;AACnE,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAE3D,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,gCAAgC,CAAC;AAC5E,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,UAAU,CAAC;AAC9C,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAE3C,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAIhD,MAAM,MAAM,iBAAiB,GAAG;IAC9B,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE;QAAE,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAAA;KAAE,CAAC;CAC9C,CAAC;AAEF,wBAAgB,eAAe,CAAC,EAAE,EAAE,OAAO,GAAG,iBAAiB,CAM9D;AAED,MAAM,MAAM,WAAW,GACnB,WAAW,GACX,iBAAiB,GACjB,iBAAiB,GACjB,WAAW,CAAC;AAEhB,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;CAChB,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,OAAO,CAAC;IAChB,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,cAAc,CAAC,EAAE,OAAO,CAAC;CAC1B,GAAG,iBAAiB,CAAC;AAEtB,MAAM,MAAM,iBAAiB,GAAG;IAC9B,MAAM,EAAE,SAAS,CAAC;IAClB,EAAE,EAAE,OAAO,CAAC;IACZ,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,QAAQ,EAAE,eAAe,CAAC;IAC1B,GAAG,EAAE;QACH,CAAC,SAAS,EAAE,SAAS,GAAG,iBAAiB,CAAC;KAC3C,CAAC;IACF,kBAAkB,CAAC,EAAE;QACnB,CAAC,SAAS,EAAE,SAAS,GAAG,MAAM,CAAC;KAChC,CAAC;CACH,CAAC;AAEF,MAAM,MAAM,iBAAiB,GAAG;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,eAAe,EAAE,WAAW,EAAE,CAAC;IAC/B,aAAa,EAAE,SAAS,CAAC;CAC1B,CAAC;AACF,MAAM,MAAM,WAAW,GAAG;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,EAAE,EAAE,OAAO,CAAC;CACb,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG,MAAM,CAAC;AAE5B,MAAM,MAAM,iBAAiB,GAAG,cAAc,CAAC;AAE/C,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,SAAS,EAAE,CAAC,QAAQ,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,KAAK,IAAI,CAAC;IAC9E,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,mBAAmB;IAClC,IAAI,EAAE,CAAC,GAAG,EAAE,WAAW,GAAG,iBAAiB,KAAK,IAAI,CAAC;IACrD,KAAK,EAAE,MAAM,IAAI,CAAC;IAClB,OAAO,EAAE,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,IAAI,CAAC;CACzC;AAED,MAAM,WAAW,IAAI;IACnB,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAC;IAC1B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,sBAAsB,CAAC,EAAE,OAAO,CAAC;CAClC;AAED,wBAAgB,mBAAmB,CACjC,MAAM,EAAE,iBAAiB,EACzB,MAAM,EAAE,iBAAiB,GACxB,iBAAiB,CAoBnB;AAED,qBAAa,WAAW;IACtB,KAAK,EAAE;QAAE,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAAA;KAAE,CAAM;IACzC,KAAK,EAAE,SAAS,CAAC;IAEjB,YAAY,sFAIT;IACH,OAAO,CAAC,yBAAyB,CAAY;gBAEjC,KAAK,EAAE,SAAS;IAa5B,SAAS,EAAE,gBAAgB,CAAC;IAE5B,oBAAoB,IAAI,SAAS,EAAE;IASnC,QAAQ,IAAI,SAAS,EAAE;IAIvB,cAAc,CAAC,aAAa,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE;IAOnD,iBAAiB,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAuCnD,mCAAmC,CACjC,EAAE,EAAE,OAAO,EACX,IAAI,EAAE,SAAS,EACf,IAAI,GAAE,GAAG,CAAC,OAAO,CAAa;IAsChC,uBAAuB,CAAC,IAAI,EAAE,SAAS;IA+DvC,aAAa,wBAA+B;IAC5C,WAAW,CAAC,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAYlD,OAAO,CAAC,IAAI,EAAE,IAAI;IAyClB,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,GAAG,EAAE,WAAW;IAI/C;;;;;;;OAOG;IACH,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS;IAyC5C,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAkBxD,sBAAsB,CAAC,eAAe,EAAE,WAAW,EAAE,EAAE,MAAM,EAAE,MAAM;IAarE,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS,GAAG,SAAS;IAsQpE,gBAAgB,CAAC,GAAG,EAAE,iBAAiB,EAAE,IAAI,EAAE,SAAS;IAMxD,cAAc,uBAAsB;IACpC,kBAAkB,CAAC,OAAO,EAAE,WAAW;IAcvC,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,IAAI,EAAE,iBAAiB,EAAE,GAAG,SAAS;IA0BxE,WAAW,CAAC,OAAO,EAAE,WAAW;IA8BhC,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM;IA4ChE,kBAAkB,CAAC,EAAE,EAAE,OAAO;IAI9B,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,SAAS;IAUzC,sBAAsB,CAAC,OAAO,SAAS;IAavC,gBAAgB;CAKjB"}

package/dist/sync.js

@@ -1,4 +1,5 @@
 import { ValueType, metrics } from "@opentelemetry/api";
+import { IncomingMessagesQueue } from "./IncomingMessagesQueue.js";
 import { PeerState } from "./PeerState.js";
 import { SyncStateManager } from "./SyncStateManager.js";
 import { getDependedOnCoValuesFromRawData } from "./coValueCore/utils.js";
@@ -37,6 +38,7 @@ export class SyncManager {
             valueType: ValueType.INT,
             unit: "peer",
         });
+        this.messagesQueue = new IncomingMessagesQueue();
         this.requestedSyncs = new Set();
         this.local = local;
         this.syncState = new SyncStateManager(this);
@@ -58,13 +60,8 @@ export class SyncManager {
     getPeers() {
         return Object.values(this.peers);
     }
-    getServerAndStoragePeers(excludePeerId) {
-        return this.peersInPriorityOrder().filter((peer) => peer.isServerOrStoragePeer() &&
-            peer.id !== excludePeerId &&
-            !peer.closed);
-    }
-    hasStoragePeers() {
-        return this.getPeers().some((peer) => peer.role === "storage" && !peer.closed);
+    getServerPeers(excludePeerId) {
+        return this.getPeers().filter((peer) => peer.role === "server" && peer.id !== excludePeerId && !peer.closed);
     }
     handleSyncMessage(msg, peer) {
         if (msg.id === undefined || msg.id === null) {
@@ -97,7 +94,7 @@ export class SyncManager {
             case "content":
                 return this.handleNewContent(msg, peer);
             case "done":
-                return this.handleUnsubscribe(msg);
+                return;
             default:
                 throw new Error(`Unknown message type ${msg.action}`);
         }
@@ -124,7 +121,7 @@ export class SyncManager {
         else if (!peer.toldKnownState.has(id)) {
             this.trySendToPeer(peer, {
                 action: "known",
-                ...coValue.knownState(),
+                ...coValue.knownStateWithStreaming(),
             });
         }
         peer.trackToldKnownState(id);
@@ -183,6 +180,15 @@ export class SyncManager {
             });
         }
     }
+    pushMessage(incoming, peer) {
+        this.messagesQueue.push(incoming, peer);
+        if (this.messagesQueue.processing) {
+            return;
+        }
+        this.messagesQueue.processQueue((msg, peer) => {
+            this.handleSyncMessage(msg, peer);
+        });
+    }
     addPeer(peer) {
         const prevPeer = this.peers[peer.id];
         if (prevPeer && !prevPeer.closed) {
@@ -194,36 +200,17 @@ export class SyncManager {
         const unsubscribeFromKnownStatesUpdates = peerState.knownStates.subscribe((id) => {
             this.syncState.triggerUpdate(peer.id, id);
         });
-        if (peerState.isServerOrStoragePeer()) {
+        if (peerState.role === "server") {
             void this.startPeerReconciliation(peerState);
         }
-        peerState
-            .processIncomingMessages((msg) => {
-            this.handleSyncMessage(msg, peerState);
-        })
-            .then(() => {
-            if (peer.crashOnClose) {
-                logger.error("Unexepcted close from peer", {
-                    peerId: peer.id,
-                    peerRole: peer.role,
-                });
-                this.local.crashed = new Error("Unexpected close from peer");
-                throw new Error("Unexpected close from peer");
-            }
-        })
-            .catch((e) => {
-            logger.error("Error processing messages from peer", {
-                err: e,
-                peerId: peer.id,
-                peerRole: peer.role,
-            });
-            if (peer.crashOnClose) {
-                this.local.crashed = e;
-                throw new Error(e);
+        peerState.incoming.onMessage((msg) => {
+            if (msg === "Disconnected") {
+                peerState.gracefulShutdown();
+                return;
             }
-        })
-            .finally(() => {
-            peerState.gracefulShutdown();
+            this.pushMessage(msg, peerState);
+        });
+        peerState.addCloseListener(() => {
             unsubscribeFromKnownStatesUpdates();
             this.peersCounter.add(-1, { role: peer.role });
             if (peer.deletePeerStateOnClose && this.peers[peer.id] === peerState) {
@@ -255,11 +242,13 @@ export class SyncManager {
             this.sendNewContentIncludingDependencies(msg.id, peer);
             return;
         }
-        const eligiblePeers = this.getServerAndStoragePeers(peer.id);
-        if (eligiblePeers.length === 0) {
-            // We don't have any eligible peers to load the coValue from
-            // so we send a known state back to the sender to let it know
-            // that the coValue is unavailable
+        const peers = this.getServerPeers(peer.id);
+        coValue.load(peers);
+        const handleLoadResult = () => {
+            if (coValue.isAvailable()) {
+                this.sendNewContentIncludingDependencies(msg.id, peer);
+                return;
+            }
             peer.trackToldKnownState(msg.id);
             this.trySendToPeer(peer, {
                 action: "known",
@@ -267,35 +256,13 @@ export class SyncManager {
                 header: false,
                 sessions: {},
             });
-            return;
+        };
+        if (peers.length > 0 || this.local.storage) {
+            coValue.waitForAvailableOrUnavailable().then(handleLoadResult);
+        }
+        else {
+            handleLoadResult();
         }
-        coValue.loadFromPeers(eligiblePeers).catch((e) => {
-            logger.error("Error loading coValue in handleLoad", { err: e });
-        });
-        // We need to return from handleLoad immediately and wait for the CoValue to be loaded
-        // in a new task, otherwise we might block further incoming content messages that would
-        // resolve the CoValue as available. This can happen when we receive fresh
-        // content from a client, but we are a server with our own upstream server(s)
-        coValue
-            .waitForAvailableOrUnavailable()
-            .then((value) => {
-            if (!value.isAvailable()) {
-                peer.trackToldKnownState(msg.id);
-                this.trySendToPeer(peer, {
-                    action: "known",
-                    id: msg.id,
-                    header: false,
-                    sessions: {},
-                });
-                return;
-            }
-            this.sendNewContentIncludingDependencies(msg.id, peer);
-        })
-            .catch((e) => {
-            logger.error("Error loading coValue in handleLoad loading state", {
-                err: e,
-            });
-        });
     }
     handleKnownState(msg, peer) {
         const coValue = this.local.getCoValue(msg.id);
@@ -320,40 +287,56 @@ export class SyncManager {
             });
         }
     }
-    handleNewContent(msg, peer) {
+    handleNewContent(msg, from) {
         const coValue = this.local.getCoValue(msg.id);
-        if (!coValue.verified) {
+        const peer = from === "storage" ? undefined : from;
+        if (!coValue.hasVerifiedContent()) {
             if (!msg.header) {
-                this.trySendToPeer(peer, {
-                    action: "known",
-                    isCorrection: true,
-                    id: msg.id,
-                    header: false,
-                    sessions: {},
-                });
+                if (peer) {
+                    this.trySendToPeer(peer, {
+                        action: "known",
+                        isCorrection: true,
+                        id: msg.id,
+                        header: false,
+                        sessions: {},
+                    });
+                }
+                else {
+                    logger.error("Received new content with no header on a missing CoValue", {
+                        id: msg.id,
+                    });
+                }
                 return;
             }
             const sessionIDs = Object.keys(msg.new);
             const transactions = Object.values(msg.new).map((content) => content.newTransactions);
             for (const dependency of getDependedOnCoValuesFromRawData(msg.id, msg.header, sessionIDs, transactions)) {
                 const dependencyCoValue = this.local.getCoValue(dependency);
-                if (!dependencyCoValue.verified) {
+                if (!dependencyCoValue.hasVerifiedContent()) {
                     coValue.markMissingDependency(dependency);
-                    if (!dependencyCoValue.verified) {
-                        const peers = this.getServerAndStoragePeers();
-                        // if the peer that sent the content is a client, we add it to the list of peers
-                        // to also ask them for the dependency
-                        if (peer.role === "client") {
-                            peers.push(peer);
-                        }
-                        dependencyCoValue.loadFromPeers(peers);
+                    const peers = this.getServerPeers();
+                    // if the peer that sent the content is a client, we add it to the list of peers
+                    // to also ask them for the dependency
+                    if (peer?.role === "client") {
+                        peers.push(peer);
                     }
+                    dependencyCoValue.load(peers);
+                }
+                else if (!dependencyCoValue.isAvailable()) {
+                    coValue.markMissingDependency(dependency);
                 }
             }
-            peer.updateHeader(msg.id, true);
-            coValue.provideHeader(msg.header, peer.id);
+            peer?.updateHeader(msg.id, true);
+            coValue.provideHeader(msg.header, peer?.id ?? "storage", msg.expectContentUntil);
+            if (msg.expectContentUntil) {
+                peer?.combineWith(msg.id, {
+                    id: msg.id,
+                    header: true,
+                    sessions: msg.expectContentUntil,
+                });
+            }
         }
-        if (!coValue.verified) {
+        if (!coValue.hasVerifiedContent()) {
             throw new Error("Unreachable: CoValue should always have a verified state at this point");
         }
         let invalidStateAssumed = false;
@@ -379,13 +362,13 @@ export class SyncManager {
                     // This covers the case where we are getting a new session on an already loaded coValue
                     // where we need to load the account to get their public key
                     if (!coValue.missingDependencies.has(accountId)) {
-                        const peers = this.getServerAndStoragePeers();
-                        if (peer.role === "client") {
+                        const peers = this.getServerPeers();
+                        if (peer?.role === "client") {
                             // if the peer that sent the content is a client, we add it to the list of peers
                             // to also ask them for the dependency
                             peers.push(peer);
                         }
-                        account.loadFromPeers(peers);
+                        account.load(peers);
                     }
                     // We need to wait for the account to be available before we can verify the transaction
                     // Currently doing this by delaying the handleNewContent for the session to when we have the account
@@ -405,35 +388,50 @@ export class SyncManager {
                                 [sessionID]: newContentForSession,
                             },
                             priority: msg.priority,
-                        }, peer);
+                        }, from);
                     });
                     continue;
                 }
             }
             const result = coValue.tryAddTransactions(sessionID, newTransactions, undefined, newContentForSession.lastSignature, "immediate");
             if (result.isErr()) {
-                console.error("Failed to add transactions", {
-                    peerId: peer.id,
-                    peerRole: peer.role,
-                    id: msg.id,
-                    err: result.error,
-                });
-                coValue.markErrored(peer.id, result.error);
+                if (peer) {
+                    logger.error("Failed to add transactions", {
+                        peerId: peer.id,
+                        peerRole: peer.role,
+                        id: msg.id,
+                        err: result.error,
+                    });
+                    coValue.markErrored(peer.id, result.error);
+                }
+                else {
+                    logger.error("Failed to add transactions from storage", {
+                        id: msg.id,
+                        err: result.error,
+                    });
+                }
                 continue;
             }
-            this.recordTransactionsSize(newTransactions, peer.role);
-            peer.updateSessionCounter(msg.id, sessionID, newContentForSession.after +
+            this.recordTransactionsSize(newTransactions, peer?.role ?? "storage");
+            peer?.updateSessionCounter(msg.id, sessionID, newContentForSession.after +
                 newContentForSession.newTransactions.length);
         }
         if (invalidStateAssumed) {
-            this.trySendToPeer(peer, {
-                action: "known",
-                isCorrection: true,
-                ...coValue.knownState(),
-            });
-            peer.trackToldKnownState(msg.id);
+            if (peer) {
+                this.trySendToPeer(peer, {
+                    action: "known",
+                    isCorrection: true,
+                    ...coValue.knownState(),
+                });
+                peer.trackToldKnownState(msg.id);
+            }
+            else {
+                logger.error("Invalid state assumed when handling new content from storage", {
+                    id: msg.id,
+                });
+            }
         }
-        else {
+        else if (peer) {
             /**
              * We are sending a known state message to the peer to acknowledge the
              * receipt of the new content.
@@ -447,18 +445,15 @@ export class SyncManager {
             });
             peer.trackToldKnownState(msg.id);
         }
-        const sourcePeer = peer;
         const syncedPeers = [];
+        if (from !== "storage") {
+            this.storeCoValue(coValue, [msg]);
+        }
         for (const peer of this.peersInPriorityOrder()) {
             /**
              * We sync the content against the source peer if it is a client or server peers
              * to upload any content that is available on the current node and not on the source peer.
-             *
-             * We don't need to do this with storage peers because we don't get updates from those peers,
-             * only load and store content.
              */
-            if (peer.id === sourcePeer.id && sourcePeer.role === "storage")
-                continue;
             if (peer.closed)
                 continue;
             if (coValue.isErroredInPeer(peer.id))
@@ -468,7 +463,7 @@ export class SyncManager {
                 this.sendNewContentIncludingDependencies(coValue.id, peer);
                 syncedPeers.push(peer);
             }
-            else if (peer.isServerOrStoragePeer() &&
+            else if (peer.role === "server" &&
                 !peer.loadRequestSent.has(coValue.id)) {
                 const state = coValue.getStateForPeer(peer.id)?.type;
                 // Check if there is a inflight load operation and we
@@ -478,7 +473,7 @@ export class SyncManager {
                     // before sending the new content
                     this.trySendToPeer(peer, {
                         action: "load",
-                        ...coValue.knownState(),
+                        ...coValue.knownStateWithStreaming(),
                     });
                     peer.trackLoadRequestSent(coValue.id);
                     syncedPeers.push(peer);
@@ -493,7 +488,6 @@ export class SyncManager {
         peer.setKnownState(msg.id, knownStateIn(msg));
         return this.sendNewContentIncludingDependencies(msg.id, peer);
     }
-    handleUnsubscribe(_msg) { }
     requestCoValueSync(coValue) {
         if (this.requestedSyncs.has(coValue.id)) {
             return;
@@ -505,8 +499,33 @@ export class SyncManager {
         });
         this.requestedSyncs.add(coValue.id);
     }
+    storeCoValue(coValue, data) {
+        const storage = this.local.storage;
+        if (!storage || !data)
+            return;
+        // Try to store the content as-is for performance
+        // In case that some transactions are missing, a correction will be requested, but it's an edge case
+        storage.store(data, (correction) => {
+            if (!coValue.hasVerifiedContent())
+                return;
+            const newContentPieces = coValue.verified.newContentSince(correction);
+            if (!newContentPieces)
+                return;
+            storage.store(newContentPieces, (response) => {
+                logger.error("Correction requested by storage after sending a correction content", {
+                    response,
+                    knownState: coValue.knownState(),
+                });
+            });
+        });
+    }
     syncCoValue(coValue) {
         this.requestedSyncs.delete(coValue.id);
+        if (this.local.storage && coValue.hasVerifiedContent()) {
+            const knownState = this.local.storage.getKnownState(coValue.id);
+            const newContentPieces = coValue.verified.newContentSince(knownState);
+            this.storeCoValue(coValue, newContentPieces);
+        }
         for (const peer of this.peersInPriorityOrder()) {
             if (peer.closed)
                 continue;
@@ -554,15 +573,14 @@ export class SyncManager {
             }, timeout);
         });
     }
-    waitForStorageSync(id, timeout = 30000) {
-        const peers = this.getPeers();
-        return Promise.all(peers
-            .filter((peer) => peer.role === "storage")
-            .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)));
+    waitForStorageSync(id) {
+        return this.local.storage?.waitForSync(id, this.local.getCoValue(id));
     }
     waitForSync(id, timeout = 30000) {
         const peers = this.getPeers();
-        return Promise.all(peers.map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout)));
+        return Promise.all(peers
+            .map((peer) => this.waitForSyncWithPeer(peer.id, id, timeout))
+            .concat(this.waitForStorageSync(id)));
     }
     waitForAllCoValuesSync(timeout = 60000) {
         const coValues = this.local.allCoValues();