cojson 0.7.35-unique.2 → 0.8.0

package/src/priority.ts

@@ -0,0 +1,39 @@
+import { type CoValueHeader } from "./coValueCore.js";
+
+/**
+ * The priority of a `CoValue` determines how much priority is given
+ * to its content messages.
+ * 
+ * The priority value is handled as weight in the weighed round robin algorithm
+ * used to determine the order in which messages are sent.
+ * 
+ * Follows the HTTP urgency range and order:
+ *  - https://www.rfc-editor.org/rfc/rfc9218.html#name-urgency
+ */
+export const CO_VALUE_PRIORITY = {
+    HIGH: 0,
+    MEDIUM: 3,
+    LOW: 6,
+} as const;
+
+export type CoValuePriority = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7;
+
+export function getPriorityFromHeader(header: CoValueHeader | undefined | boolean): CoValuePriority {
+    if (typeof header === "boolean" || !header) {
+        return CO_VALUE_PRIORITY.MEDIUM;
+    }
+
+    if (header.meta?.type === "account") {
+        return CO_VALUE_PRIORITY.HIGH;
+    }
+
+    if (header.ruleset.type === "group") {
+        return CO_VALUE_PRIORITY.HIGH;
+    }
+
+    if (header.type === "costream" && header.meta?.type === "binary") {
+        return CO_VALUE_PRIORITY.LOW;
+    }
+
+    return CO_VALUE_PRIORITY.MEDIUM;
+}

package/src/storage/chunksAndKnownStates.ts

@@ -1,5 +1,6 @@
 import { RawCoID, SessionID } from "../ids.js";
 import { MAX_RECOMMENDED_TX_SIZE } from "../index.js";
+import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { CoValueChunk } from "./index.js";
 
@@ -15,6 +16,7 @@ export function contentSinceChunk(
         action: "content",
         header: known?.header ? undefined : chunk.header,
         new: {},
+        priority: getPriorityFromHeader(chunk.header),
     });
 
     for (const [sessionID, sessionsEntry] of Object.entries(

package/src/sync.ts

@@ -3,6 +3,8 @@ import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
 import { LocalNode, newLoadingState } from "./localNode.js";
 import { RawCoID, SessionID } from "./ids.js";
+import { PeerState } from "./PeerState.js";
+import { CoValuePriority } from "./priority.js";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -38,6 +40,7 @@ export type NewContentMessage = {
     action: "content";
     id: RawCoID;
     header?: CoValueHeader;
+    priority: CoValuePriority;
     new: {
         [sessionID: SessionID]: SessionNewContent;
     };
@@ -76,17 +79,6 @@ export interface Peer {
     crashOnClose: boolean;
 }
 
-export interface PeerState {
-    id: PeerID;
-    optimisticKnownStates: { [id: RawCoID]: CoValueKnownState };
-    toldKnownState: Set<RawCoID>;
-    incoming: IncomingSyncStream;
-    outgoing: OutgoingSyncQueue;
-    role: "peer" | "server" | "client";
-    priority?: number;
-    crashOnClose: boolean;
-}
-
 export function combinedKnownStates(
     stateA: CoValueKnownState,
     stateB: CoValueKnownState,
@@ -141,7 +133,7 @@ export class SyncManager {
 
         for (const peer of eligiblePeers) {
             // console.log("loading", id, "from", peer.id);
-            await peer.outgoing.push({
+            await peer.pushOutgoingMessage({
                 action: "load",
                 id: id,
                 header: false,
@@ -227,7 +219,7 @@ export class SyncManager {
                 id,
                 header: false,
                 sessions: {},
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending load", e);
             });
             return;
@@ -244,7 +236,7 @@ export class SyncManager {
             this.trySendToPeer(peer, {
                 action: "load",
                 ...coValue.knownState(),
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending load", e);
             });
         }
@@ -274,7 +266,7 @@ export class SyncManager {
                 action: "known",
                 asDependencyOf,
                 ...coValue.knownState(),
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending known state", e);
             });
 
@@ -282,7 +274,10 @@ export class SyncManager {
         }
     }
 
-    async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
+    async sendNewContentIncludingDependencies(
+        id: RawCoID,
+        peer: PeerState,
+    ) {
         const coValue = this.local.expectCoValueLoaded(id);
 
         await Promise.all(
@@ -310,9 +305,11 @@ export class SyncManager {
                     //     } header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
-                    this.trySendToPeer(peer, piece).catch((e) => {
+
+                    this.trySendToPeer(peer, piece).catch((e: unknown) => {
                         console.error("Error sending content piece", e);
                     });
+
                     if (performance.now() - lastYield > 10) {
                         await new Promise<void>((resolve) => {
                             setTimeout(resolve, 0);
@@ -336,16 +333,7 @@ export class SyncManager {
     }
 
     addPeer(peer: Peer) {
-        const peerState: PeerState = {
-            id: peer.id,
-            optimisticKnownStates: {},
-            incoming: peer.incoming,
-            outgoing: peer.outgoing,
-            toldKnownState: new Set(),
-            role: peer.role,
-            priority: peer.priority,
-            crashOnClose: peer.crashOnClose,
-        };
+        const peerState = new PeerState(peer);
         this.peers[peer.id] = peerState;
 
         if (peer.role === "server") {
@@ -420,7 +408,7 @@ export class SyncManager {
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        return peer.outgoing.push(msg);
+        return peer.pushOutgoingMessage(msg);
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
@@ -679,7 +667,8 @@ export class SyncManager {
                     newTransactions.length + " new transactions",
                     "after: " + newContentForSession.after,
                     "our last known tx idx initially: " + ourKnownTxIdx,
-                    "our last known tx idx now: " + coValue.sessionLogs.get(sessionID)?.transactions.length,
+                    "our last known tx idx now: " +
+                        coValue.sessionLogs.get(sessionID)?.transactions.length,
                 );
                 continue;
             }
@@ -774,11 +763,7 @@ export class SyncManager {
 
     gracefulShutdown() {
         for (const peer of Object.values(this.peers)) {
-            console.debug("Gracefully closing", peer.id);
-            peer.outgoing.close();
-            peer.incoming = (async function* () {
-                yield "Disconnected" as const;
-            })();
+            peer.gracefulShutdown();
         }
     }
 }

package/src/tests/PeerState.test.ts

@@ -0,0 +1,92 @@
+import { describe, test, expect, vi } from "vitest";
+import { PeerState } from "../PeerState.js";
+import { Peer, SyncMessage } from "../sync.js";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+
+function setup() {
+    const mockPeer: Peer = {
+        id: "test-peer",
+        role: "peer",
+        priority: 1,
+        crashOnClose: false,
+        incoming: (async function* () {})(),
+        outgoing: {
+            push: vi.fn().mockResolvedValue(undefined),
+            close: vi.fn(),
+        },
+    };
+    const peerState = new PeerState(mockPeer);
+    return { mockPeer, peerState };
+}
+
+describe("PeerState", () => {
+    test("should initialize with correct properties", () => {
+        const { peerState } = setup();
+        expect(peerState.id).toBe("test-peer");
+        expect(peerState.role).toBe("peer");
+        expect(peerState.priority).toBe(1);
+        expect(peerState.crashOnClose).toBe(false);
+        expect(peerState.closed).toBe(false);
+        expect(peerState.optimisticKnownStates).toEqual({});
+        expect(peerState.toldKnownState).toEqual(new Set());
+    });
+
+    test("should push outgoing message to peer", async () => {
+        const { mockPeer, peerState } = setup();
+        const message: SyncMessage = { action: "load", id: "co_ztest-id", header: false, sessions: {} };
+        await peerState.pushOutgoingMessage(message);
+        expect(mockPeer.outgoing.push).toHaveBeenCalledWith(message);
+    });
+
+    test("should return peer's incoming when not closed", () => {
+        const { mockPeer, peerState } = setup();
+        expect(peerState.incoming).toBe(mockPeer.incoming);
+    });
+
+    test("should return Disconnected when closed", async () => {
+        const { peerState } = setup();
+        peerState.gracefulShutdown();
+        const incomingIterator = peerState.incoming[Symbol.asyncIterator]();
+        const { value, done } = await incomingIterator.next();
+        expect(value).toBe("Disconnected");
+        expect(done).toBe(false);
+    });
+
+    test("should perform graceful shutdown", () => {
+        const { mockPeer, peerState } = setup();
+        const consoleSpy = vi.spyOn(console, "debug").mockImplementation(() => {});
+        peerState.gracefulShutdown();
+        expect(mockPeer.outgoing.close).toHaveBeenCalled();
+        expect(peerState.closed).toBe(true);
+        expect(consoleSpy).toHaveBeenCalledWith("Gracefully closing", "test-peer");
+        consoleSpy.mockRestore();
+    });
+
+    test("should schedule outgoing messages based on their priority", async () => {
+        const { peerState } = setup();
+
+        const loadMessage: SyncMessage = { action: "load", id: "co_zhigh", header: false, sessions: {} };
+        const contentMessageHigh: SyncMessage = { action: "content", id: "co_zhigh", new: {}, priority: CO_VALUE_PRIORITY.HIGH };
+        const contentMessageMid: SyncMessage = { action: "content", id: "co_zmid", new: {}, priority: CO_VALUE_PRIORITY.MEDIUM };
+        const contentMessageLow: SyncMessage = { action: "content", id: "co_zlow", new: {}, priority: CO_VALUE_PRIORITY.LOW };
+
+        const promises = [
+            peerState.pushOutgoingMessage(contentMessageLow),
+            peerState.pushOutgoingMessage(contentMessageMid),
+            peerState.pushOutgoingMessage(contentMessageHigh),
+            peerState.pushOutgoingMessage(loadMessage),
+        ];
+
+        await Promise.all(promises);
+
+        // The first message is pushed directly, the other three are queued because are waiting
+        // for the first push to be completed.
+        expect(peerState["peer"].outgoing.push).toHaveBeenNthCalledWith(1, contentMessageLow); 
+
+        // Load message are managed as high priority messages and having the same priority as the content message
+        // they follow the push order.
+        expect(peerState["peer"].outgoing.push).toHaveBeenNthCalledWith(2, contentMessageHigh);
+        expect(peerState["peer"].outgoing.push).toHaveBeenNthCalledWith(3, loadMessage);
+        expect(peerState["peer"].outgoing.push).toHaveBeenNthCalledWith(4, contentMessageMid);
+    });
+});

package/src/tests/PriorityBasedMessageQueue.test.ts

@@ -0,0 +1,111 @@
+import { describe, test, expect } from "vitest";
+import { PriorityBasedMessageQueue } from "../PriorityBasedMessageQueue.js";
+import { SyncMessage } from "../sync.js";
+import { CO_VALUE_PRIORITY } from "../priority.js";
+
+function setup() {
+    const queue = new PriorityBasedMessageQueue(CO_VALUE_PRIORITY.MEDIUM);
+    return { queue };
+}
+
+describe("PriorityBasedMessageQueue", () => {
+    test("should initialize with correct properties", () => {
+        const { queue } = setup();
+        expect(queue["defaultPriority"]).toBe(CO_VALUE_PRIORITY.MEDIUM);
+        expect(queue["queues"].length).toBe(8);
+        expect(queue["queues"].every((q) => q.length === 0)).toBe(true);
+    });
+
+    test("should push message with default priority", async () => {
+        const { queue } = setup();
+        const message: SyncMessage = {
+            action: "load",
+            id: "co_ztest-id",
+            header: false,
+            sessions: {},
+        };
+        void queue.push(message);
+        const pulledEntry = queue.pull();
+        expect(pulledEntry?.msg).toEqual(message);
+    });
+
+    test("should push message with specified priority", async () => {
+        const { queue } = setup();
+        const message: SyncMessage = {
+            action: "content",
+            id: "co_zhigh",
+            new: {},
+            priority: CO_VALUE_PRIORITY.HIGH,
+        };
+        void queue.push(message);
+        const pulledEntry = queue.pull();
+        expect(pulledEntry?.msg).toEqual(message);
+    });
+
+    test("should pull messages in priority order", async () => {
+        const { queue } = setup();
+        const lowPriorityMsg: SyncMessage = {
+            action: "content",
+            id: "co_zlow",
+            new: {},
+            priority: CO_VALUE_PRIORITY.LOW,
+        };
+        const mediumPriorityMsg: SyncMessage = {
+            action: "content",
+            id: "co_zmedium",
+            new: {},
+            priority: CO_VALUE_PRIORITY.MEDIUM,
+        };
+        const highPriorityMsg: SyncMessage = {
+            action: "content",
+            id: "co_zhigh",
+            new: {},
+            priority: CO_VALUE_PRIORITY.HIGH,
+        };
+
+        void queue.push(lowPriorityMsg);
+        void queue.push(mediumPriorityMsg);
+        void queue.push(highPriorityMsg);
+
+        expect(queue.pull()?.msg).toEqual(highPriorityMsg);
+        expect(queue.pull()?.msg).toEqual(mediumPriorityMsg);
+        expect(queue.pull()?.msg).toEqual(lowPriorityMsg);
+    });
+
+    test("should return undefined when pulling from empty queue", () => {
+        const { queue } = setup();
+        expect(queue.pull()).toBeUndefined();
+    });
+
+    test("should resolve promise when message is pulled", async () => {
+        const { queue } = setup();
+        const message: SyncMessage = {
+            action: "load",
+            id: "co_ztest-id",
+            header: false,
+            sessions: {},
+        };
+        const pushPromise = queue.push(message);
+
+        const pulledEntry = queue.pull();
+        pulledEntry?.resolve();
+
+        await expect(pushPromise).resolves.toBeUndefined();
+    });
+
+    test("should reject promise when message is rejected", async () => {
+        const { queue } = setup();
+        const message: SyncMessage = {
+            action: "load",
+            id: "co_ztest-id",
+            header: false,
+            sessions: {},
+        };
+        const pushPromise = queue.push(message);
+
+        const pulledEntry = queue.pull();
+        pulledEntry?.reject(new Error("Test error"));
+
+        await expect(pushPromise).rejects.toThrow("Test error");
+    });
+});

package/src/tests/account.test.ts

@@ -1,5 +1,4 @@
 import { expect, test } from "vitest";
-import { newRandomSessionID } from "../coValueCore.js";
 import { LocalNode } from "../localNode.js";
 import { connectedPeers } from "../streamUtils.js";
 import { WasmCrypto } from "../crypto/WasmCrypto.js";
@@ -65,7 +64,7 @@ test("Can create account with one node, and then load it on another", async () =
     const node2 = await LocalNode.withLoadedAccount({
         accountID,
         accountSecret,
-        sessionID: newRandomSessionID(accountID),
+        sessionID: Crypto.newRandomSessionID(accountID),
         peersToLoadFrom: [node1asPeer],
         crypto: Crypto,
     });

package/src/tests/coStream.test.ts

@@ -1,4 +1,4 @@
-import { expect, test } from "vitest";
+import { expect, test, describe } from "vitest";
 import { expectStream } from "../coValue.js";
 import { RawBinaryCoStream } from "../coValues/coStream.js";
 import { MAX_RECOMMENDED_TX_SIZE, WasmCrypto } from "../index.js";
@@ -244,3 +244,60 @@ test("When adding large transactions (bigger than MAX_RECOMMENDED_TX_SIZE), we s
         sessionEntry.lastSignature,
     );
 });
+
+describe("isBinaryStreamEnded", () => {
+    function setup() {
+        const node = new LocalNode(
+            ...randomAnonymousAccountAndSessionID(),
+            Crypto,
+        );
+
+        const coValue = node.createCoValue({
+            type: "costream",
+            ruleset: { type: "unsafeAllowAll" },
+            meta: { type: "binary" },
+            ...Crypto.createdNowUnique(),
+        });
+
+        const content = coValue.getCurrentContent();
+
+        if (
+            content.type !== "costream" ||
+            content.headerMeta?.type !== "binary" ||
+            !(content instanceof RawBinaryCoStream)
+        ) {
+            throw new Error("Expected binary stream");
+        }
+
+        return content;
+    }
+
+    test("returns true when the last item is end", () => {
+        const stream = setup();
+
+        stream.startBinaryStream(
+            { mimeType: "text/plain", fileName: "test.txt" },
+            "trusting",
+        );
+        stream.endBinaryStream("trusting");
+
+        expect(stream.isBinaryStreamEnded()).toBe(true);
+    });
+
+    test("returns false if the stream isn't ended", () => {
+        const stream = setup();
+
+        stream.startBinaryStream(
+            { mimeType: "text/plain", fileName: "test.txt" },
+            "trusting",
+        );
+
+        expect(stream.isBinaryStreamEnded()).toBe(false);
+    });
+
+    test("returns false if the stream isn't started", () => {
+        const stream = setup();
+
+        expect(stream.isBinaryStreamEnded()).toBe(false);
+    });
+});