cojson 0.4.8 → 0.5.0

package/src/streamUtils.ts

@@ -18,11 +18,11 @@ export function connectedPeers(
         peer2role?: Peer["role"];
     } = {}
 ): [Peer, Peer] {
-    const [inRx1, inTx1] = newStreamPair<SyncMessage>();
-    const [outRx1, outTx1] = newStreamPair<SyncMessage>();
+    const [inRx1, inTx1] = newStreamPair<SyncMessage>(peer1id + "_in");
+    const [outRx1, outTx1] = newStreamPair<SyncMessage>(peer1id + "_out");
 
-    const [inRx2, inTx2] = newStreamPair<SyncMessage>();
-    const [outRx2, outTx2] = newStreamPair<SyncMessage>();
+    const [inRx2, inTx2] = newStreamPair<SyncMessage>(peer2id + "_in");
+    const [outRx2, outTx2] = newStreamPair<SyncMessage>(peer2id + "_out");
 
     void outRx2
         .pipeThrough(
@@ -37,7 +37,7 @@ export function connectedPeers(
                             JSON.stringify(
                                 chunk,
                                 (k, v) =>
-                                    (k === "changes" || k === "encryptedChanges")
+                                    k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
                                 2
@@ -62,7 +62,7 @@ export function connectedPeers(
                             JSON.stringify(
                                 chunk,
                                 (k, v) =>
-                                    (k === "changes" || k === "encryptedChanges")
+                                    k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
                                 2
@@ -91,7 +91,10 @@ export function connectedPeers(
     return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
+export function newStreamPair<T>(
+    pairName?: string
+): [ReadableStream<T>, WritableStream<T>] {
+    let queueLength = 0;
     let readerClosed = false;
 
     let resolveEnqueue: (enqueue: (item: T) => void) => void;
@@ -104,6 +107,22 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
         resolveClose = resolve;
     });
 
+    let queueWasOverflowing = false;
+
+    function maybeReportQueueLength() {
+        if (queueLength >= 100) {
+            queueWasOverflowing = true;
+            if (queueLength % 100 === 0) {
+                console.warn(pairName, "overflowing queue length", queueLength);
+            }
+        } else {
+            if (queueWasOverflowing) {
+                console.debug(pairName, "ok queue length", queueLength);
+                queueWasOverflowing = false;
+            }
+        }
+    }
+
     const readable = new ReadableStream<T>({
         async start(controller) {
             resolveEnqueue(controller.enqueue.bind(controller));
@@ -114,12 +133,26 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
             console.log("Manually closing reader");
             readerClosed = true;
         },
-    });
+    }).pipeThrough(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        new TransformStream<any, any>({
+            transform(
+                chunk: SyncMessage,
+                controller: { enqueue: (msg: SyncMessage) => void }
+            ) {
+                queueLength -= 1;
+                maybeReportQueueLength();
+                controller.enqueue(chunk);
+            },
+        })
+    ) as ReadableStream<T>;
 
     let lastWritePromise = Promise.resolve();
 
     const writable = new WritableStream<T>({
         async write(chunk) {
+            queueLength += 1;
+            maybeReportQueueLength();
             const enqueue = await enqueuePromise;
             if (readerClosed) {
                 throw new Error("Reader closed");

package/src/sync.ts

@@ -2,7 +2,6 @@ import { Signature } from "./crypto.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
 import { LocalNode } from "./localNode.js";
-import { newLoadingState } from "./localNode.js";
 import {
     ReadableStream,
     WritableStream,
@@ -67,6 +66,7 @@ export interface Peer {
     outgoing: WritableStream<SyncMessage>;
     role: "peer" | "server" | "client";
     delayOnError?: number;
+    priority?: number;
 }
 
 export interface PeerState {
@@ -77,6 +77,7 @@ export interface PeerState {
     outgoing: WritableStreamDefaultWriter<SyncMessage>;
     role: "peer" | "server" | "client";
     delayOnError?: number;
+    priority?: number;
 }
 
 export function combinedKnownStates(
@@ -107,13 +108,30 @@ export function combinedKnownStates(
 export class SyncManager {
     peers: { [key: PeerID]: PeerState } = {};
     local: LocalNode;
+    requestedSyncs: { [id: RawCoID]: {done: Promise<void>, nRequestsThisTick: number} | undefined } = {};
 
     constructor(local: LocalNode) {
         this.local = local;
     }
 
-    loadFromPeers(id: RawCoID) {
-        for (const peer of Object.values(this.peers)) {
+    peersInPriorityOrder(): PeerState[] {
+        return Object.values(this.peers).sort((a, b) => {
+            const aPriority = a.priority || 0;
+            const bPriority = b.priority || 0;
+
+            return bPriority - aPriority;
+        });
+    }
+
+    async loadFromPeers(id: RawCoID, excludePeer?: PeerID) {
+        for (const peer of this.peersInPriorityOrder()) {
+            if (peer.id === excludePeer) {
+                continue;
+            }
+            if (peer.role !== "server") {
+                continue;
+            }
+            // console.log("loading", id, "from", peer.id);
             peer.outgoing
                 .write({
                     action: "load",
@@ -124,6 +142,44 @@ export class SyncManager {
                 .catch((e) => {
                     console.error("Error writing to peer", e);
                 });
+            const coValueEntry = this.local.coValues[id];
+            if (coValueEntry?.state !== "loading") {
+                continue;
+            }
+            const firstStateEntry = coValueEntry.firstPeerState[peer.id];
+            if (firstStateEntry?.type !== "waiting") {
+                throw new Error("Expected firstPeerState to be waiting " + id);
+            }
+            await new Promise<void>((resolve) => {
+                const timeout = setTimeout(() => {
+                    if (this.local.coValues[id]?.state === "loading") {
+                        // console.warn(
+                        //     "Timeout waiting for peer to load",
+                        //     id,
+                        //     "from",
+                        //     peer.id,
+                        //     "and it hasn't loaded from other peers yet"
+                        // );
+                    }
+                    resolve();
+                }, 1000);
+                firstStateEntry.done
+                    .then(() => {
+                        clearTimeout(timeout);
+                        resolve();
+                    })
+                    .catch((e) => {
+                        clearTimeout(timeout);
+                        console.error(
+                            "Error waiting for peer to load",
+                            id,
+                            "from",
+                            peer.id,
+                            e
+                        );
+                        resolve();
+                    });
+            });
         }
     }
 
@@ -139,6 +195,7 @@ export class SyncManager {
                     return await this.handleKnownState(msg, peer);
                 }
             case "content":
+                await new Promise<void>((resolve) => setTimeout(resolve, 0));
                 return await this.handleNewContent(msg, peer);
             case "done":
                 return await this.handleUnsubscribe(msg);
@@ -190,13 +247,11 @@ export class SyncManager {
     ) {
         const coValue = this.local.expectCoValueLoaded(id);
 
-        for (const dependentCoID of coValue.getDependedOnCoValues()) {
-            await this.tellUntoldKnownStateIncludingDependencies(
-                dependentCoID,
-                peer,
-                asDependencyOf || id
-            );
-        }
+        await Promise.all(coValue.getDependedOnCoValues().map(dependentCoID => this.tellUntoldKnownStateIncludingDependencies(
+            dependentCoID,
+            peer,
+            asDependencyOf || id
+        )));
 
         if (!peer.toldKnownState.has(id)) {
             await this.trySendToPeer(peer, {
@@ -212,9 +267,7 @@ export class SyncManager {
     async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
         const coValue = this.local.expectCoValueLoaded(id);
 
-        for (const id of coValue.getDependedOnCoValues()) {
-            await this.sendNewContentIncludingDependencies(id, peer);
-        }
+        await Promise.all(coValue.getDependedOnCoValues().map(id => this.sendNewContentIncludingDependencies(id, peer)));
 
         const newContentPieces = coValue.newContentSince(
             peer.optimisticKnownStates[id]
@@ -225,12 +278,19 @@ export class SyncManager {
                 peer.optimisticKnownStates[id] || emptyKnownState(id);
 
             const sendPieces = async () => {
+                let lastYield = performance.now();
                 for (const [_i, piece] of newContentPieces.entries()) {
                     // console.log(
                     //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
                     await this.trySendToPeer(peer, piece);
+                    if (performance.now() - lastYield > 10) {
+                        await new Promise<void>((resolve) => {
+                            setTimeout(resolve, 0);
+                        });
+                        lastYield = performance.now();
+                    }
                 }
             };
 
@@ -256,6 +316,7 @@ export class SyncManager {
             toldKnownState: new Set(),
             role: peer.role,
             delayOnError: peer.delayOnError,
+            priority: peer.priority,
         };
         this.peers[peer.id] = peerState;
 
@@ -264,6 +325,7 @@ export class SyncManager {
                 for (const id of Object.keys(
                     this.local.coValues
                 ) as RawCoID[]) {
+                    // console.log("subscribing to after peer added", id, peer.id)
                     await this.subscribeToIncludingDependencies(id, peerState);
 
                     peerState.optimisticKnownStates[id] = {
@@ -280,7 +342,19 @@ export class SyncManager {
             try {
                 for await (const msg of peerState.incoming) {
                     try {
-                        await this.handleSyncMessage(msg, peerState);
+                        // await this.handleSyncMessage(msg, peerState);
+                        this.handleSyncMessage(msg, peerState).catch((e) => {
+                            console.error(
+                                new Date(),
+                                `Error reading from peer ${peer.id}, handling msg`,
+                                JSON.stringify(msg, (k, v) =>
+                                    k === "changes" || k === "encryptedChanges"
+                                        ? v.slice(0, 20) + "..."
+                                        : v
+                                ),
+                                e
+                            );
+                        });
                         await new Promise<void>((resolve) => {
                             setTimeout(resolve, 0);
                         });
@@ -314,20 +388,28 @@ export class SyncManager {
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
+        if (!this.peers[peer.id]) {
+            // already disconnected, return to drain potential queue
+            return Promise.resolve();
+        }
+
         return new Promise<void>((resolve) => {
-            const timeout = setTimeout(() => {
-                console.error(
-                    new Error(
-                        `Writing to peer ${peer.id} took >1s - this should never happen as write should resolve quickly or error`
-                    )
-                );
-                resolve();
-            }, 1000);
+            const start = Date.now();
             peer.outgoing
                 .write(msg)
                 .then(() => {
-                    clearTimeout(timeout);
-                    resolve();
+                    const end = Date.now();
+                    if (end - start > 1000) {
+                        // console.error(
+                        //     new Error(
+                        //         `Writing to peer "${peer.id}" took ${
+                        //             Math.round((Date.now() - start) / 100) / 10
+                        //         }s - this should never happen as write should resolve quickly or error`
+                        //     )
+                        // );
+                    } else {
+                        resolve();
+                    }
                 })
                 .catch((e) => {
                     console.error(
@@ -344,42 +426,42 @@ export class SyncManager {
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
-        const entry = this.local.coValues[msg.id];
+        peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
+        let entry = this.local.coValues[msg.id];
 
-        if (!entry || entry.state === "loading") {
-            if (!entry) {
-                await new Promise<void>((resolve) => {
-                    this.local
-                        .loadCoValue(msg.id)
-                        .then(() => resolve())
-                        .catch((e) => {
-                            console.error(
-                                "Error loading coValue in handleLoad",
-                                e
-                            );
-                            resolve();
-                        });
-                    setTimeout(resolve, 1000);
+        if (!entry) {
+            // console.log(`Loading ${msg.id} from all peers except ${peer.id}`);
+            this.local
+                .loadCoValueCore(msg.id, {
+                    dontLoadFrom: peer.id,
+                    dontWaitFor: peer.id,
+                })
+                .catch((e) => {
+                    console.error("Error loading coValue in handleLoad", e);
                 });
-            }
 
-            peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
-            peer.toldKnownState.add(msg.id);
+            entry = this.local.coValues[msg.id]!;
+        }
 
-            await this.trySendToPeer(peer, {
-                action: "known",
-                id: msg.id,
-                header: false,
-                sessions: {},
-            });
+        if (entry.state === "loading") {
+            const loaded = await entry.done;
 
-            return;
-        }
+            if (loaded === "unavailable") {
+                peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
+                peer.toldKnownState.add(msg.id);
 
-        peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
+                await this.trySendToPeer(peer, {
+                    action: "known",
+                    id: msg.id,
+                    header: false,
+                    sessions: {},
+                });
 
-        await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
+                return;
+            }
+        }
 
+        await this.tellUntoldKnownStateIncludingDependencies(msg.id, peer);
         await this.sendNewContentIncludingDependencies(msg.id, peer);
     }
 
@@ -394,9 +476,15 @@ export class SyncManager {
         if (!entry) {
             if (msg.asDependencyOf) {
                 if (this.local.coValues[msg.asDependencyOf]) {
-                    entry = newLoadingState();
-
-                    this.local.coValues[msg.id] = entry;
+                    this.local
+                        .loadCoValueCore(msg.id, { dontLoadFrom: peer.id })
+                        .catch((e) => {
+                            console.error(
+                                `Error loading coValue ${msg.id} to create loading state, as dependency of ${msg.asDependencyOf}`,
+                                e
+                            );
+                        });
+                    entry = this.local.coValues[msg.id]!; // must exist after loadCoValueCore
                 } else {
                     throw new Error(
                         "Expected coValue dependency entry to be created, missing subscribe?"
@@ -410,6 +498,29 @@ export class SyncManager {
         }
 
         if (entry.state === "loading") {
+            const availableOnPeer = peer.optimisticKnownStates[msg.id]?.header;
+            const firstPeerStateEntry = entry.firstPeerState[peer.id];
+            if (firstPeerStateEntry?.type === "waiting") {
+                firstPeerStateEntry.resolve();
+            }
+            entry.firstPeerState[peer.id] = availableOnPeer
+                ? { type: "available" }
+                : { type: "unavailable" };
+            // console.log(
+            //     "Marking",
+            //     msg.id,
+            //     "as",
+            //     entry.firstPeerState[peer.id]?.type,
+            //     "from",
+            //     peer.id
+            // );
+            if (
+                Object.values(entry.firstPeerState).every(
+                    (s) => s.type === "unavailable"
+                )
+            ) {
+                entry.resolve("unavailable");
+            }
             return [];
         }
 
@@ -441,6 +552,12 @@ export class SyncManager {
                 throw new Error("Expected header to be sent in first message");
             }
 
+            const firstPeerStateEntry = entry.firstPeerState[peer.id];
+            if (firstPeerStateEntry?.type === "waiting") {
+                firstPeerStateEntry.resolve();
+                entry.firstPeerState[peer.id] = { type: "available" };
+            }
+
             peerOptimisticKnownState.header = true;
 
             const coValue = new CoValueCore(msg.header, this.local);
@@ -450,6 +567,7 @@ export class SyncManager {
             entry = {
                 state: "loaded",
                 coValue: coValue,
+                onProgress: entry.onProgress,
             };
 
             this.local.coValues[msg.id] = entry;
@@ -463,7 +581,7 @@ export class SyncManager {
             msg.new
         ) as [SessionID, SessionNewContent][]) {
             const ourKnownTxIdx =
-                coValue.sessions[sessionID]?.transactions.length;
+                coValue.sessionLogs.get(sessionID)?.transactions.length;
             const theirFirstNewTxIdx = newContentForSession.after;
 
             if ((ourKnownTxIdx || 0) < theirFirstNewTxIdx) {
@@ -490,7 +608,7 @@ export class SyncManager {
                 newContentForSession.lastSignature
             );
             const after = performance.now();
-            if (after - before > 10) {
+            if (after - before > 80) {
                 const totalTxLength = newTransactions
                     .map((t) =>
                         t.privacy === "private"
@@ -509,6 +627,16 @@ export class SyncManager {
                 );
             }
 
+            const theirTotalnTxs = Object.values(
+                peer.optimisticKnownStates[msg.id]?.sessions || {}
+            ).reduce((sum, nTxs) => sum + nTxs, 0);
+            const ourTotalnTxs = [...coValue.sessionLogs.values()].reduce(
+                (sum, session) => sum + session.transactions.length,
+                0
+            );
+
+            entry.onProgress?.(ourTotalnTxs / theirTotalnTxs);
+
             if (!success) {
                 console.error(
                     "Failed to add transactions",
@@ -522,9 +650,11 @@ export class SyncManager {
                 continue;
             }
 
-            peerOptimisticKnownState.sessions[sessionID] =
+            peerOptimisticKnownState.sessions[sessionID] = Math.max(
+                peerOptimisticKnownState.sessions[sessionID] || 0,
                 newContentForSession.after +
-                newContentForSession.newTransactions.length;
+                    newContentForSession.newTransactions.length
+            );
         }
 
         if (resolveAfterDone) {
@@ -553,7 +683,38 @@ export class SyncManager {
     }
 
     async syncCoValue(coValue: CoValueCore) {
-        for (const peer of Object.values(this.peers)) {
+        if (this.requestedSyncs[coValue.id]) {
+            this.requestedSyncs[coValue.id]!.nRequestsThisTick++;
+            return this.requestedSyncs[coValue.id]!.done;
+        } else {
+            const done = new Promise<void>((resolve) => {
+                setTimeout(async () => {
+                    delete this.requestedSyncs[coValue.id];
+                    if (entry.nRequestsThisTick >= 2) {
+                        console.log("Syncing", coValue.id, "for", entry.nRequestsThisTick, "requests");
+                    }
+                    await this.actuallySyncCoValue(coValue);
+                    resolve();
+                }, 0);
+            });
+            const entry = {
+                done,
+                nRequestsThisTick: 1,
+            };
+            this.requestedSyncs[coValue.id] = entry;
+            return done;
+        }
+    }
+
+    async actuallySyncCoValue(coValue: CoValueCore) {
+        let blockingSince = performance.now();
+        for (const peer of this.peersInPriorityOrder()) {
+            if (performance.now() - blockingSince > 5) {
+                await new Promise<void>((resolve) => {
+                    setTimeout(resolve, 0);
+                });
+                blockingSince = performance.now();
+            }
             const optimisticKnownState = peer.optimisticKnownStates[coValue.id];
 
             if (optimisticKnownState) {

package/src/tests/coValue.test.ts

@@ -1,9 +1,9 @@
 import { expectList, expectMap, expectStream } from "../coValue.js";
-import { accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { BinaryCoStream } from "../coValues/coStream.js";
 import { createdNowUnique } from "../crypto.js";
 import { MAX_RECOMMENDED_TX_SIZE, cojsonReady } from "../index.js";
 import { LocalNode } from "../localNode.js";
+import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
 import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
 
 beforeEach(async () => {
@@ -423,7 +423,7 @@ test("When adding large transactions (small fraction of MAX_RECOMMENDED_TX_SIZE)
         editable.endBinaryStream("trusting");
     });
 
-    const sessionEntry = coValue._sessions[node.currentSessionID]!;
+    const sessionEntry = coValue.sessionLogs.get(node.currentSessionID)!;
     expect(sessionEntry.transactions.length).toEqual(12);
     expect(sessionEntry.signatureAfter[0]).not.toBeDefined();
     expect(sessionEntry.signatureAfter[1]).not.toBeDefined();
@@ -499,7 +499,7 @@ test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we s
         editable.endBinaryStream("trusting");
     });
 
-    const sessionEntry = coValue._sessions[node.currentSessionID]!;
+    const sessionEntry = coValue.sessionLogs.get(node.currentSessionID)!;
     expect(sessionEntry.transactions.length).toEqual(5);
     expect(sessionEntry.signatureAfter[0]).not.toBeDefined();
     expect(sessionEntry.signatureAfter[1]).toBeDefined();