cojson 0.7.11 → 0.7.14

package/src/sync.ts

@@ -1,13 +1,9 @@
 import { Signature } from "./crypto/crypto.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
-import { LocalNode } from "./localNode.js";
-import {
-    ReadableStream,
-    WritableStream,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
+import { LocalNode, newLoadingState } from "./localNode.js";
 import { RawCoID, SessionID } from "./ids.js";
+import { Effect, Queue, Stream } from "effect";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -60,10 +56,27 @@ export type DoneMessage = {
 
 export type PeerID = string;
 
+export class DisconnectedError extends Error {
+    readonly _tag = "DisconnectedError";
+    constructor(public message: string) {
+        super(message);
+    }
+}
+
+export class PingTimeoutError extends Error {
+    readonly _tag = "PingTimeoutError";
+}
+
+export type IncomingSyncStream = Stream.Stream<
+    SyncMessage,
+    DisconnectedError | PingTimeoutError
+>;
+export type OutgoingSyncQueue = Queue.Enqueue<SyncMessage>;
+
 export interface Peer {
     id: PeerID;
-    incoming: ReadableStream<SyncMessage>;
-    outgoing: WritableStream<SyncMessage>;
+    incoming: IncomingSyncStream;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -73,8 +86,8 @@ export interface PeerState {
     id: PeerID;
     optimisticKnownStates: { [id: RawCoID]: CoValueKnownState };
     toldKnownState: Set<RawCoID>;
-    incoming: ReadableStream<SyncMessage>;
-    outgoing: WritableStreamDefaultWriter<SyncMessage>;
+    incoming: IncomingSyncStream;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -127,25 +140,24 @@ export class SyncManager {
         });
     }
 
-    async loadFromPeers(id: RawCoID, excludePeer?: PeerID) {
-        for (const peer of this.peersInPriorityOrder()) {
-            if (peer.id === excludePeer) {
-                continue;
-            }
-            if (peer.role !== "server") {
-                continue;
-            }
+    async loadFromPeers(id: RawCoID, forPeer?: PeerID) {
+        const eligiblePeers = this.peersInPriorityOrder().filter(
+            (peer) => peer.id !== forPeer && peer.role === "server",
+        );
+
+        for (const peer of eligiblePeers) {
             // console.log("loading", id, "from", peer.id);
-            peer.outgoing
-                .write({
+            Effect.runPromise(
+                Queue.offer(peer.outgoing, {
                     action: "load",
                     id: id,
                     header: false,
                     sessions: {},
-                })
-                .catch((e) => {
-                    console.error("Error writing to peer", e);
-                });
+                }),
+            ).catch((e) => {
+                console.error("Error writing to peer", e);
+            });
+
             const coValueEntry = this.local.coValues[id];
             if (coValueEntry?.state !== "loading") {
                 continue;
@@ -297,7 +309,9 @@ export class SyncManager {
                 let lastYield = performance.now();
                 for (const [_i, piece] of newContentPieces.entries()) {
                     // console.log(
-                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
+                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${
+                    //         newContentPieces.length
+                    //     } header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
                     await this.trySendToPeer(peer, piece);
@@ -328,7 +342,7 @@ export class SyncManager {
             id: peer.id,
             optimisticKnownStates: {},
             incoming: peer.incoming,
-            outgoing: peer.outgoing.getWriter(),
+            outgoing: peer.outgoing,
             toldKnownState: new Set(),
             role: peer.role,
             delayOnError: peer.delayOnError,
@@ -354,91 +368,55 @@ export class SyncManager {
             void initialSync();
         }
 
-        const readIncoming = async () => {
-            try {
-                for await (const msg of peerState.incoming) {
-                    try {
-                        // await this.handleSyncMessage(msg, peerState);
-                        this.handleSyncMessage(msg, peerState).catch((e) => {
-                            console.error(
-                                new Date(),
-                                `Error reading from peer ${peer.id}, handling msg`,
-                                JSON.stringify(msg, (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                ),
-                                e,
-                            );
-                        });
-                        // await new Promise<void>((resolve) => {
-                        //     setTimeout(resolve, 0);
-                        // });
-                    } catch (e) {
-                        console.error(
-                            new Date(),
-                            `Error reading from peer ${peer.id}, handling msg`,
-                            JSON.stringify(msg, (k, v) =>
-                                k === "changes" || k === "encryptedChanges"
-                                    ? v.slice(0, 20) + "..."
-                                    : v,
+        void Effect.runPromise(
+            peerState.incoming.pipe(
+                Stream.ensuring(
+                    Effect.sync(() => {
+                        console.log("Peer disconnected:", peer.id);
+                        delete this.peers[peer.id];
+                    }),
+                ),
+                Stream.runForEach((msg) =>
+                    Effect.tryPromise({
+                        try: () => this.handleSyncMessage(msg, peerState),
+                        catch: (e) =>
+                            new Error(
+                                `Error reading from peer ${
+                                    peer.id
+                                }, handling msg\n\n${JSON.stringify(
+                                    msg,
+                                    (k, v) =>
+                                        k === "changes" ||
+                                        k === "encryptedChanges"
+                                            ? v.slice(0, 20) + "..."
+                                            : v,
+                                )}`,
+                                { cause: e },
                             ),
-                            e,
-                        );
-                        if (peerState.delayOnError) {
-                            await new Promise<void>((resolve) => {
-                                setTimeout(resolve, peerState.delayOnError);
-                            });
-                        }
-                    }
-                }
-            } catch (e) {
-                console.error(`Error reading from peer ${peer.id}`, e);
-            }
-
-            console.log("Peer disconnected:", peer.id);
-            delete this.peers[peer.id];
-        };
-
-        void readIncoming();
+                    }).pipe(
+                        Effect.timeoutFail({
+                            duration: 10000,
+                            onTimeout: () =>
+                                new Error("Took >10s to process message"),
+                        }),
+                    ),
+                ),
+                Effect.catchAll((e) =>
+                    Effect.logError(
+                        "Error in peer",
+                        peer.id,
+                        e.message,
+                        typeof e.cause === "object" &&
+                            e.cause instanceof Error &&
+                            e.cause.message,
+                    ),
+                ),
+            ),
+        );
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        if (!this.peers[peer.id]) {
-            // already disconnected, return to drain potential queue
-            return Promise.resolve();
-        }
-
-        return new Promise<void>((resolve) => {
-            const start = Date.now();
-            peer.outgoing
-                .write(msg)
-                .then(() => {
-                    const end = Date.now();
-                    if (end - start > 1000) {
-                        // console.error(
-                        //     new Error(
-                        //         `Writing to peer "${peer.id}" took ${
-                        //             Math.round((Date.now() - start) / 100) / 10
-                        //         }s - this should never happen as write should resolve quickly or error`
-                        //     )
-                        // );
-                    } else {
-                        resolve();
-                    }
-                })
-                .catch((e) => {
-                    console.error(
-                        new Error(
-                            `Error writing to peer ${peer.id}, disconnecting`,
-                            {
-                                cause: e,
-                            },
-                        ),
-                    );
-                    delete this.peers[peer.id];
-                });
-        });
+        return Effect.runPromise(Queue.offer(peer.outgoing, msg));
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
@@ -447,21 +425,50 @@ export class SyncManager {
 
         if (!entry) {
             // console.log(`Loading ${msg.id} from all peers except ${peer.id}`);
-            this.local
-                .loadCoValueCore(msg.id, {
-                    dontLoadFrom: peer.id,
-                    dontWaitFor: peer.id,
-                })
-                .catch((e) => {
-                    console.error("Error loading coValue in handleLoad", e);
-                });
+
+            // special case: we should be able to solve this much more neatly
+            // with an explicit state machine in the future
+            const eligiblePeers = this.peersInPriorityOrder().filter(
+                (other) => other.id !== peer.id && peer.role === "server",
+            );
+            if (eligiblePeers.length === 0) {
+                if (msg.header || Object.keys(msg.sessions).length > 0) {
+                    this.local.coValues[msg.id] = newLoadingState(
+                        new Set([peer.id]),
+                    );
+                    this.trySendToPeer(peer, {
+                        action: "known",
+                        id: msg.id,
+                        header: false,
+                        sessions: {},
+                    }).catch((e) => {
+                        console.error("Error sending known state back", e);
+                    });
+                }
+                return;
+            } else {
+                this.local
+                    .loadCoValueCore(msg.id, {
+                        dontLoadFrom: peer.id,
+                        dontWaitFor: peer.id,
+                    })
+                    .catch((e) => {
+                        console.error("Error loading coValue in handleLoad", e);
+                    });
+            }
 
             entry = this.local.coValues[msg.id]!;
         }
 
         if (entry.state === "loading") {
+            console.log(
+                "Waiting for loaded",
+                msg.id,
+                "after message from",
+                peer.id,
+            );
             const loaded = await entry.done;
-
+            console.log("Loaded", msg.id, loaded);
             if (loaded === "unavailable") {
                 peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
                 peer.toldKnownState.add(msg.id);
@@ -508,7 +515,7 @@ export class SyncManager {
                 }
             } else {
                 throw new Error(
-                    "Expected coValue entry to be created, missing subscribe?",
+                    `Expected coValue entry for ${msg.id} to be created on known state, missing subscribe?`,
                 );
             }
         }
@@ -549,7 +556,7 @@ export class SyncManager {
 
         if (!entry) {
             throw new Error(
-                "Expected coValue entry to be created, missing subscribe?",
+                `Expected coValue entry for ${msg.id} to be created on new content, missing subscribe?`,
             );
         }
 

package/src/tests/account.test.ts

@@ -3,6 +3,7 @@ import { newRandomSessionID } from "../coValueCore.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { Effect } from "effect";
 
 const Crypto = await WasmCrypto.create();
 
@@ -52,11 +53,13 @@ test("Can create account with one node, and then load it on another", async () =
     map.set("foo", "bar", "private");
     expect(map.get("foo")).toEqual("bar");
 
-    const [node1asPeer, node2asPeer] = connectedPeers("node1", "node2", {
+    const [node1asPeer, node2asPeer] = await Effect.runPromise(connectedPeers("node1", "node2", {
         trace: true,
         peer1role: "server",
         peer2role: "client",
-    });
+    }));
+
+    console.log("After connected peers")
 
     node.syncManager.addPeer(node2asPeer);