cojson 0.7.11 → 0.7.17

package/src/streamUtils.ts

@@ -1,8 +1,4 @@
-import {
-    ReadableStream,
-    TransformStream,
-    WritableStream,
-} from "isomorphic-streams";
+import { Console, Effect, Queue, Stream } from "effect";
 import { Peer, PeerID, SyncMessage } from "./sync.js";
 
 export function connectedPeers(
@@ -17,160 +13,54 @@ export function connectedPeers(
         peer1role?: Peer["role"];
         peer2role?: Peer["role"];
     } = {},
-): [Peer, Peer] {
-    const [inRx1, inTx1] = newStreamPair<SyncMessage>(peer1id + "_in");
-    const [outRx1, outTx1] = newStreamPair<SyncMessage>(peer1id + "_out");
-
-    const [inRx2, inTx2] = newStreamPair<SyncMessage>(peer2id + "_in");
-    const [outRx2, outTx2] = newStreamPair<SyncMessage>(peer2id + "_out");
-
-    void outRx2
-        .pipeThrough(
-            new TransformStream({
-                transform(
-                    chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void },
-                ) {
-                    trace &&
-                        console.debug(
-                            `${peer2id} -> ${peer1id}`,
-                            JSON.stringify(
-                                chunk,
-                                (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                2,
-                            ),
-                        );
-                    controller.enqueue(chunk);
-                },
-            }),
-        )
-        .pipeTo(inTx1);
-
-    void outRx1
-        .pipeThrough(
-            new TransformStream({
-                transform(
-                    chunk: SyncMessage,
-                    controller: { enqueue: (msg: SyncMessage) => void },
-                ) {
-                    trace &&
-                        console.debug(
-                            `${peer1id} -> ${peer2id}`,
-                            JSON.stringify(
-                                chunk,
-                                (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                2,
-                            ),
-                        );
-                    controller.enqueue(chunk);
-                },
-            }),
-        )
-        .pipeTo(inTx2);
-
-    const peer2AsPeer: Peer = {
-        id: peer2id,
-        incoming: inRx1,
-        outgoing: outTx1,
-        role: peer2role,
-    };
-
-    const peer1AsPeer: Peer = {
-        id: peer1id,
-        incoming: inRx2,
-        outgoing: outTx2,
-        role: peer1role,
-    };
-
-    return [peer1AsPeer, peer2AsPeer];
-}
-
-export function newStreamPair<T>(
-    pairName?: string,
-): [ReadableStream<T>, WritableStream<T>] {
-    let queueLength = 0;
-    let readerClosed = false;
-
-    let resolveEnqueue: (enqueue: (item: T) => void) => void;
-    const enqueuePromise = new Promise<(item: T) => void>((resolve) => {
-        resolveEnqueue = resolve;
+): Effect.Effect<[Peer, Peer]> {
+    return Effect.gen(function* () {
+        const [from1to2Rx, from1to2Tx] = yield* newQueuePair(
+            trace ? { traceAs: `${peer1id} -> ${peer2id}` } : undefined,
+        );
+        const [from2to1Rx, from2to1Tx] = yield* newQueuePair(
+            trace ? { traceAs: `${peer2id} -> ${peer1id}` } : undefined,
+        );
+
+        const peer2AsPeer: Peer = {
+            id: peer2id,
+            incoming: from2to1Rx,
+            outgoing: from1to2Tx,
+            role: peer2role,
+        };
+
+        const peer1AsPeer: Peer = {
+            id: peer1id,
+            incoming: from1to2Rx,
+            outgoing: from2to1Tx,
+            role: peer1role,
+        };
+
+        return [peer1AsPeer, peer2AsPeer];
     });
+}
 
-    let resolveClose: (close: () => void) => void;
-    const closePromise = new Promise<() => void>((resolve) => {
-        resolveClose = resolve;
-    });
-
-    let queueWasOverflowing = false;
-
-    function maybeReportQueueLength() {
-        if (queueLength >= 100) {
-            queueWasOverflowing = true;
-            if (queueLength % 100 === 0) {
-                console.warn(pairName, "overflowing queue length", queueLength);
-            }
+export function newQueuePair(
+    options: { traceAs?: string } = {},
+): Effect.Effect<[Stream.Stream<SyncMessage>, Queue.Enqueue<SyncMessage>]> {
+    return Effect.gen(function* () {
+        const queue = yield* Queue.unbounded<SyncMessage>();
+
+        if (options.traceAs) {
+            return [Stream.fromQueue(queue).pipe(Stream.tap((msg) => Console.debug(
+                options.traceAs,
+                JSON.stringify(
+                    msg,
+                    (k, v) =>
+                        k === "changes" ||
+                        k === "encryptedChanges"
+                            ? v.slice(0, 20) + "..."
+                            : v,
+                    2,
+                ),
+            ))), queue];
         } else {
-            if (queueWasOverflowing) {
-                console.debug(pairName, "ok queue length", queueLength);
-                queueWasOverflowing = false;
-            }
+            return [Stream.fromQueue(queue), queue];
         }
-    }
-
-    const readable = new ReadableStream<T>({
-        async start(controller) {
-            resolveEnqueue(controller.enqueue.bind(controller));
-            resolveClose(controller.close.bind(controller));
-        },
-
-        cancel(_reason) {
-            console.log("Manually closing reader");
-            readerClosed = true;
-        },
-    }).pipeThrough(
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        new TransformStream<any, any>({
-            transform(
-                chunk: SyncMessage,
-                controller: { enqueue: (msg: SyncMessage) => void },
-            ) {
-                queueLength -= 1;
-                maybeReportQueueLength();
-                controller.enqueue(chunk);
-            },
-        }),
-    ) as ReadableStream<T>;
-
-    let lastWritePromise = Promise.resolve();
-
-    const writable = new WritableStream<T>({
-        async write(chunk) {
-            queueLength += 1;
-            maybeReportQueueLength();
-            const enqueue = await enqueuePromise;
-            if (readerClosed) {
-                throw new Error("Reader closed");
-            } else {
-                // make sure write resolves before corresponding read, but make sure writes are still in order
-                await lastWritePromise;
-                lastWritePromise = new Promise((resolve) => {
-                    enqueue(chunk);
-                    resolve();
-                });
-            }
-        },
-        async abort(reason) {
-            console.debug("Manually closing writer", reason);
-            const close = await closePromise;
-            close();
-        },
     });
-
-    return [readable, writable];
 }

package/src/sync.ts

@@ -1,13 +1,9 @@
 import { Signature } from "./crypto/crypto.js";
 import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
-import { LocalNode } from "./localNode.js";
-import {
-    ReadableStream,
-    WritableStream,
-    WritableStreamDefaultWriter,
-} from "isomorphic-streams";
+import { LocalNode, newLoadingState } from "./localNode.js";
 import { RawCoID, SessionID } from "./ids.js";
+import { Effect, Queue, Stream } from "effect";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -60,10 +56,27 @@ export type DoneMessage = {
 
 export type PeerID = string;
 
+export class DisconnectedError extends Error {
+    readonly _tag = "DisconnectedError";
+    constructor(public message: string) {
+        super(message);
+    }
+}
+
+export class PingTimeoutError extends Error {
+    readonly _tag = "PingTimeoutError";
+}
+
+export type IncomingSyncStream = Stream.Stream<
+    SyncMessage,
+    DisconnectedError | PingTimeoutError
+>;
+export type OutgoingSyncQueue = Queue.Enqueue<SyncMessage>;
+
 export interface Peer {
     id: PeerID;
-    incoming: ReadableStream<SyncMessage>;
-    outgoing: WritableStream<SyncMessage>;
+    incoming: IncomingSyncStream;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -73,8 +86,8 @@ export interface PeerState {
     id: PeerID;
     optimisticKnownStates: { [id: RawCoID]: CoValueKnownState };
     toldKnownState: Set<RawCoID>;
-    incoming: ReadableStream<SyncMessage>;
-    outgoing: WritableStreamDefaultWriter<SyncMessage>;
+    incoming: IncomingSyncStream;
+    outgoing: OutgoingSyncQueue;
     role: "peer" | "server" | "client";
     delayOnError?: number;
     priority?: number;
@@ -127,25 +140,24 @@ export class SyncManager {
         });
     }
 
-    async loadFromPeers(id: RawCoID, excludePeer?: PeerID) {
-        for (const peer of this.peersInPriorityOrder()) {
-            if (peer.id === excludePeer) {
-                continue;
-            }
-            if (peer.role !== "server") {
-                continue;
-            }
+    async loadFromPeers(id: RawCoID, forPeer?: PeerID) {
+        const eligiblePeers = this.peersInPriorityOrder().filter(
+            (peer) => peer.id !== forPeer && peer.role === "server",
+        );
+
+        for (const peer of eligiblePeers) {
             // console.log("loading", id, "from", peer.id);
-            peer.outgoing
-                .write({
+            Effect.runPromise(
+                Queue.offer(peer.outgoing, {
                     action: "load",
                     id: id,
                     header: false,
                     sessions: {},
-                })
-                .catch((e) => {
-                    console.error("Error writing to peer", e);
-                });
+                }),
+            ).catch((e) => {
+                console.error("Error writing to peer", e);
+            });
+
             const coValueEntry = this.local.coValues[id];
             if (coValueEntry?.state !== "loading") {
                 continue;
@@ -297,7 +309,9 @@ export class SyncManager {
                 let lastYield = performance.now();
                 for (const [_i, piece] of newContentPieces.entries()) {
                     // console.log(
-                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
+                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${
+                    //         newContentPieces.length
+                    //     } header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
                     await this.trySendToPeer(peer, piece);
@@ -328,7 +342,7 @@ export class SyncManager {
             id: peer.id,
             optimisticKnownStates: {},
             incoming: peer.incoming,
-            outgoing: peer.outgoing.getWriter(),
+            outgoing: peer.outgoing,
             toldKnownState: new Set(),
             role: peer.role,
             delayOnError: peer.delayOnError,
@@ -354,91 +368,55 @@ export class SyncManager {
             void initialSync();
         }
 
-        const readIncoming = async () => {
-            try {
-                for await (const msg of peerState.incoming) {
-                    try {
-                        // await this.handleSyncMessage(msg, peerState);
-                        this.handleSyncMessage(msg, peerState).catch((e) => {
-                            console.error(
-                                new Date(),
-                                `Error reading from peer ${peer.id}, handling msg`,
-                                JSON.stringify(msg, (k, v) =>
-                                    k === "changes" || k === "encryptedChanges"
-                                        ? v.slice(0, 20) + "..."
-                                        : v,
-                                ),
-                                e,
-                            );
-                        });
-                        // await new Promise<void>((resolve) => {
-                        //     setTimeout(resolve, 0);
-                        // });
-                    } catch (e) {
-                        console.error(
-                            new Date(),
-                            `Error reading from peer ${peer.id}, handling msg`,
-                            JSON.stringify(msg, (k, v) =>
-                                k === "changes" || k === "encryptedChanges"
-                                    ? v.slice(0, 20) + "..."
-                                    : v,
+        void Effect.runPromise(
+            peerState.incoming.pipe(
+                Stream.ensuring(
+                    Effect.sync(() => {
+                        console.log("Peer disconnected:", peer.id);
+                        delete this.peers[peer.id];
+                    }),
+                ),
+                Stream.runForEach((msg) =>
+                    Effect.tryPromise({
+                        try: () => this.handleSyncMessage(msg, peerState),
+                        catch: (e) =>
+                            new Error(
+                                `Error reading from peer ${
+                                    peer.id
+                                }, handling msg\n\n${JSON.stringify(
+                                    msg,
+                                    (k, v) =>
+                                        k === "changes" ||
+                                        k === "encryptedChanges"
+                                            ? v.slice(0, 20) + "..."
+                                            : v,
+                                )}`,
+                                { cause: e },
                             ),
-                            e,
-                        );
-                        if (peerState.delayOnError) {
-                            await new Promise<void>((resolve) => {
-                                setTimeout(resolve, peerState.delayOnError);
-                            });
-                        }
-                    }
-                }
-            } catch (e) {
-                console.error(`Error reading from peer ${peer.id}`, e);
-            }
-
-            console.log("Peer disconnected:", peer.id);
-            delete this.peers[peer.id];
-        };
-
-        void readIncoming();
+                    }).pipe(
+                        Effect.timeoutFail({
+                            duration: 10000,
+                            onTimeout: () =>
+                                new Error("Took >10s to process message"),
+                        }),
+                    ),
+                ),
+                Effect.catchAll((e) =>
+                    Effect.logError(
+                        "Error in peer",
+                        peer.id,
+                        e.message,
+                        typeof e.cause === "object" &&
+                            e.cause instanceof Error &&
+                            e.cause.message,
+                    ),
+                ),
+            ),
+        );
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        if (!this.peers[peer.id]) {
-            // already disconnected, return to drain potential queue
-            return Promise.resolve();
-        }
-
-        return new Promise<void>((resolve) => {
-            const start = Date.now();
-            peer.outgoing
-                .write(msg)
-                .then(() => {
-                    const end = Date.now();
-                    if (end - start > 1000) {
-                        // console.error(
-                        //     new Error(
-                        //         `Writing to peer "${peer.id}" took ${
-                        //             Math.round((Date.now() - start) / 100) / 10
-                        //         }s - this should never happen as write should resolve quickly or error`
-                        //     )
-                        // );
-                    } else {
-                        resolve();
-                    }
-                })
-                .catch((e) => {
-                    console.error(
-                        new Error(
-                            `Error writing to peer ${peer.id}, disconnecting`,
-                            {
-                                cause: e,
-                            },
-                        ),
-                    );
-                    delete this.peers[peer.id];
-                });
-        });
+        return Effect.runPromise(Queue.offer(peer.outgoing, msg));
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
@@ -447,21 +425,50 @@ export class SyncManager {
 
         if (!entry) {
             // console.log(`Loading ${msg.id} from all peers except ${peer.id}`);
-            this.local
-                .loadCoValueCore(msg.id, {
-                    dontLoadFrom: peer.id,
-                    dontWaitFor: peer.id,
-                })
-                .catch((e) => {
-                    console.error("Error loading coValue in handleLoad", e);
-                });
+
+            // special case: we should be able to solve this much more neatly
+            // with an explicit state machine in the future
+            const eligiblePeers = this.peersInPriorityOrder().filter(
+                (other) => other.id !== peer.id && peer.role === "server",
+            );
+            if (eligiblePeers.length === 0) {
+                if (msg.header || Object.keys(msg.sessions).length > 0) {
+                    this.local.coValues[msg.id] = newLoadingState(
+                        new Set([peer.id]),
+                    );
+                    this.trySendToPeer(peer, {
+                        action: "known",
+                        id: msg.id,
+                        header: false,
+                        sessions: {},
+                    }).catch((e) => {
+                        console.error("Error sending known state back", e);
+                    });
+                }
+                return;
+            } else {
+                this.local
+                    .loadCoValueCore(msg.id, {
+                        dontLoadFrom: peer.id,
+                        dontWaitFor: peer.id,
+                    })
+                    .catch((e) => {
+                        console.error("Error loading coValue in handleLoad", e);
+                    });
+            }
 
             entry = this.local.coValues[msg.id]!;
         }
 
         if (entry.state === "loading") {
+            console.log(
+                "Waiting for loaded",
+                msg.id,
+                "after message from",
+                peer.id,
+            );
             const loaded = await entry.done;
-
+            console.log("Loaded", msg.id, loaded);
             if (loaded === "unavailable") {
                 peer.optimisticKnownStates[msg.id] = knownStateIn(msg);
                 peer.toldKnownState.add(msg.id);
@@ -508,7 +515,7 @@ export class SyncManager {
                 }
             } else {
                 throw new Error(
-                    "Expected coValue entry to be created, missing subscribe?",
+                    `Expected coValue entry for ${msg.id} to be created on known state, missing subscribe?`,
                 );
             }
         }
@@ -549,7 +556,7 @@ export class SyncManager {
 
         if (!entry) {
             throw new Error(
-                "Expected coValue entry to be created, missing subscribe?",
+                `Expected coValue entry for ${msg.id} to be created on new content, missing subscribe?`,
             );
         }
 

package/src/tests/account.test.ts

@@ -3,6 +3,7 @@ import { newRandomSessionID } from "../coValueCore.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
+import { Effect } from "effect";
 
 const Crypto = await WasmCrypto.create();
 
@@ -52,11 +53,13 @@ test("Can create account with one node, and then load it on another", async () =
     map.set("foo", "bar", "private");
     expect(map.get("foo")).toEqual("bar");
 
-    const [node1asPeer, node2asPeer] = connectedPeers("node1", "node2", {
+    const [node1asPeer, node2asPeer] = await Effect.runPromise(connectedPeers("node1", "node2", {
         trace: true,
         peer1role: "server",
         peer2role: "client",
-    });
+    }));
+
+    console.log("After connected peers")
 
     node.syncManager.addPeer(node2asPeer);