cojson 0.7.35-unique.2 → 0.7.35

package/src/coValues/coStream.ts

@@ -7,7 +7,7 @@ import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfrom
 import { RawGroup } from "./group.js";
 import { AgentID, SessionID, TransactionID } from "../ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
-import { RawAccountID } from "./account.js";
+import { AccountID } from "./account.js";
 
 export type BinaryStreamInfo = {
     mimeType: string;
@@ -96,22 +96,27 @@ export class RawCoStreamView<
     }
 
     getSingleStream(): Item[] | undefined {
-        if (Object.keys(this.items).length === 0) {
+        const streams = Object.values(this.items);
+        const firstStream = streams[0];
+
+        if (!firstStream) {
             return undefined;
-        } else if (Object.keys(this.items).length !== 1) {
+        }
+
+        if (streams.length > 1) {
             throw new Error(
                 "CoStream.getSingleStream() can only be called when there is exactly one stream",
             );
         }
 
-        return Object.values(this.items)[0]?.map((item) => item.value);
+        return firstStream.map((item) => item.value);
     }
 
     sessions(): SessionID[] {
         return Object.keys(this.items) as SessionID[];
     }
 
-    accounts(): Set<RawAccountID> {
+    accounts(): Set<AccountID> {
         return new Set(
             this.sessions()
                 .map(accountOrAgentIDfromSessionID)
@@ -124,7 +129,7 @@ export class RawCoStreamView<
         n: number,
     ):
         | {
-              by: RawAccountID | AgentID;
+              by: AccountID | AgentID;
               tx: TransactionID;
               at: Date;
               value: Item;
@@ -146,7 +151,7 @@ export class RawCoStreamView<
 
     lastItemIn(sessionID: SessionID):
         | {
-              by: RawAccountID | AgentID;
+              by: AccountID | AgentID;
               tx: TransactionID;
               at: Date;
               value: Item;
@@ -170,9 +175,9 @@ export class RawCoStreamView<
         }
     }
 
-    lastItemBy(account: RawAccountID | AgentID):
+    lastItemBy(account: AccountID | AgentID):
         | {
-              by: RawAccountID | AgentID;
+              by: AccountID | AgentID;
               tx: TransactionID;
               at: Date;
               value: Item;
@@ -180,7 +185,7 @@ export class RawCoStreamView<
         | undefined {
         let latestItem:
             | {
-                  by: RawAccountID | AgentID;
+                  by: AccountID | AgentID;
                   tx: TransactionID;
                   at: Date;
                   value: Item;
@@ -205,7 +210,7 @@ export class RawCoStreamView<
         return latestItem;
     }
 
-    *itemsBy(account: RawAccountID | AgentID) {
+    *itemsBy(account: AccountID | AgentID) {
         // TODO: this can be made more lazy without a huge collect and sort
         const items = [
             ...Object.keys(this.items).flatMap((sessionID) =>
@@ -264,14 +269,26 @@ export class RawBinaryCoStreamView<
     extends RawCoStreamView<BinaryStreamItem, Meta>
     implements RawCoValue
 {
+    isBinaryStreamEnded() {
+        const items = this.getSingleStream();
+
+        if (!items || items.length === 0) {
+            return false;
+        }
+
+        const lastItem = items[items.length - 1];
+
+        return lastItem?.type === "end";
+    }
+
     getBinaryChunks(
         allowUnfinished?: boolean,
     ):
         | (BinaryStreamInfo & { chunks: Uint8Array[]; finished: boolean })
         | undefined {
-        // const before = performance.now();
         const items = this.getSingleStream();
 
+        // No active streams
         if (!items) return;
 
         const start = items[0];
@@ -288,9 +305,6 @@ export class RawBinaryCoStreamView<
         const chunks: Uint8Array[] = [];
 
         let finished = false;
-        // let totalLength = 0;
-
-        let lastProgressUpdate = Date.now();
 
         for (const item of items.slice(1)) {
             if (item.type === "end") {
@@ -306,20 +320,9 @@ export class RawBinaryCoStreamView<
             const chunk = base64URLtoBytes(
                 item.chunk.slice(binary_U_prefixLength),
             );
-            // totalLength += chunk.length;
             chunks.push(chunk);
-
-            if (Date.now() - lastProgressUpdate > 100) {
-                lastProgressUpdate = Date.now();
-            }
         }
 
-        // const after = performance.now();
-        // console.log(
-        //     "getBinaryChunks bandwidth in MB/s",
-        //     (1000 * totalLength) / (after - before) / (1024 * 1024)
-        // );
-
         return {
             mimeType: start.mimeType,
             fileName: start.fileName,

package/src/coValues/group.ts

@@ -5,10 +5,9 @@ import { JsonObject } from "../jsonValue.js";
 import { RawBinaryCoStream, RawCoStream } from "./coStream.js";
 import { Encrypted, KeyID, KeySecret, Sealed } from "../crypto/crypto.js";
 import { AgentID, isAgentID } from "../ids.js";
-import { RawAccount, RawAccountID, ControlledAccountOrAgent } from "./account.js";
+import { RawAccount, AccountID, ControlledAccountOrAgent } from "./account.js";
 import { Role } from "../permissions.js";
 import { base58 } from "@scure/base";
-import { CoValueUniqueness } from "../coValueCore.js";
 
 export const EVERYONE = "everyone" as const;
 export type Everyone = "everyone";
@@ -16,10 +15,10 @@ export type Everyone = "everyone";
 export type GroupShape = {
     profile: CoID<RawCoMap> | null;
     root: CoID<RawCoMap> | null;
-    [key: RawAccountID | AgentID]: Role;
+    [key: AccountID | AgentID]: Role;
     [EVERYONE]?: Role;
     readKey?: KeyID;
-    [revelationFor: `${KeyID}_for_${RawAccountID | AgentID}`]: Sealed<KeySecret>;
+    [revelationFor: `${KeyID}_for_${AccountID | AgentID}`]: Sealed<KeySecret>;
     [revelationFor: `${KeyID}_for_${Everyone}`]: KeySecret;
     [oldKeyForNewKey: `${KeyID}_for_${KeyID}`]: Encrypted<
         KeySecret,
@@ -56,12 +55,12 @@ export class RawGroup<
      *
      * @category 1. Role reading
      */
-    roleOf(accountID: RawAccountID): Role | undefined {
+    roleOf(accountID: AccountID): Role | undefined {
         return this.roleOfInternal(accountID);
     }
 
     /** @internal */
-    roleOfInternal(accountID: RawAccountID | AgentID): Role | undefined {
+    roleOfInternal(accountID: AccountID | AgentID): Role | undefined {
         return this.get(accountID);
     }
 
@@ -157,7 +156,7 @@ export class RawGroup<
             } else {
                 return false;
             }
-        }) as (RawAccountID | AgentID)[];
+        }) as (AccountID | AgentID)[];
 
         const maybeCurrentReadKey = this.core.getCurrentReadKey();
 
@@ -256,7 +255,6 @@ export class RawGroup<
         init?: M["_shape"],
         meta?: M["headerMeta"],
         initPrivacy: "trusting" | "private" = "private",
-        uniqueness: CoValueUniqueness = this.core.crypto.createdNowUnique()
     ): M {
         const map = this.core.node
             .createCoValue({
@@ -266,7 +264,7 @@ export class RawGroup<
                     group: this.id,
                 },
                 meta: meta || null,
-                ...uniqueness
+                ...this.core.crypto.createdNowUnique(),
             })
             .getCurrentContent() as M;
 
@@ -289,7 +287,6 @@ export class RawGroup<
         init?: L["_item"][],
         meta?: L["headerMeta"],
         initPrivacy: "trusting" | "private" = "private",
-        uniqueness: CoValueUniqueness = this.core.crypto.createdNowUnique()
     ): L {
         const list = this.core.node
             .createCoValue({
@@ -299,7 +296,7 @@ export class RawGroup<
                     group: this.id,
                 },
                 meta: meta || null,
-                ...uniqueness
+                ...this.core.crypto.createdNowUnique(),
             })
             .getCurrentContent() as L;
 
@@ -313,7 +310,7 @@ export class RawGroup<
     }
 
     /** @category 3. Value creation */
-    createStream<C extends RawCoStream>(meta?: C["headerMeta"], uniqueness: CoValueUniqueness = this.core.crypto.createdNowUnique()): C {
+    createStream<C extends RawCoStream>(meta?: C["headerMeta"]): C {
         return this.core.node
             .createCoValue({
                 type: "costream",
@@ -322,7 +319,7 @@ export class RawGroup<
                     group: this.id,
                 },
                 meta: meta || null,
-                ...uniqueness
+                ...this.core.crypto.createdNowUnique(),
             })
             .getCurrentContent() as C;
     }
@@ -330,7 +327,6 @@ export class RawGroup<
     /** @category 3. Value creation */
     createBinaryStream<C extends RawBinaryCoStream>(
         meta: C["headerMeta"] = { type: "binary" },
-        uniqueness: CoValueUniqueness = this.core.crypto.createdNowUnique()
     ): C {
         return this.core.node
             .createCoValue({
@@ -340,7 +336,7 @@ export class RawGroup<
                     group: this.id,
                 },
                 meta: meta,
-                ...uniqueness
+                ...this.core.crypto.createdNowUnique(),
             })
             .getCurrentContent() as C;
     }

package/src/ids.ts

@@ -1,4 +1,4 @@
-import { RawAccountID } from "./coValues/account.js";
+import { AccountID } from "./coValues/account.js";
 import { base58 } from "@scure/base";
 import { shortHashLength } from "./crypto/crypto.js";
 
@@ -28,4 +28,4 @@ export function isAgentID(id: string): id is AgentID {
     );
 }
 
-export type SessionID = `${RawAccountID | AgentID}_session_z${string}`;
+export type SessionID = `${AccountID | AgentID}_session_z${string}`;

package/src/index.ts

@@ -1,13 +1,12 @@
 import {
     CoValueCore,
-    type CoValueUniqueness,
     newRandomSessionID,
     MAX_RECOMMENDED_TX_SIZE,
     idforHeader,
 } from "./coValueCore.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { LocalNode } from "./localNode.js";
-import type { RawCoValue } from "./coValue.js";
+import { type RawCoValue } from "./coValue.js";
 import { RawCoMap } from "./coValues/coMap.js";
 import { RawCoList } from "./coValues/coList.js";
 import { RawCoStream, RawBinaryCoStream } from "./coValues/coStream.js";
@@ -51,7 +50,7 @@ import type {
 import { DisconnectedError, PingTimeoutError } from "./sync.js";
 import type { AgentSecret } from "./crypto/crypto.js";
 import type {
-    RawAccountID,
+    AccountID,
     AccountMeta,
     RawAccountMigration,
 } from "./coValues/account.js";
@@ -62,6 +61,7 @@ type Value = JsonValue | AnyRawCoValue;
 
 import { LSMStorage, BlockFilename, WalFilename } from "./storage/index.js";
 import { FileSystem } from "./storage/FileSystem.js";
+import { getPriorityFromHeader } from "./priority.js";
 
 /** @hidden */
 export const cojsonInternals = {
@@ -81,6 +81,7 @@ export const cojsonInternals = {
     idforHeader,
     StreamingHash,
     Channel,
+    getPriorityFromHeader,
 };
 
 export {
@@ -97,7 +98,7 @@ export {
     CoID,
     AnyRawCoValue,
     RawAccount,
-    RawAccountID,
+    AccountID,
     AccountMeta,
     RawAccountMigration,
     RawProfile as Profile,
@@ -131,7 +132,6 @@ export type {
     OutgoingSyncQueue,
     DisconnectedError,
     PingTimeoutError,
-    CoValueUniqueness
 };
 
 // eslint-disable-next-line @typescript-eslint/no-namespace

package/src/localNode.ts

@@ -2,7 +2,6 @@ import { AgentSecret, CryptoProvider } from "./crypto/crypto.js";
 import {
     CoValueCore,
     CoValueHeader,
-    CoValueUniqueness,
     newRandomSessionID,
 } from "./coValueCore.js";
 import {
@@ -20,7 +19,7 @@ import {
     ControlledAccountOrAgent,
     RawControlledAccount,
     ControlledAgent,
-    RawAccountID,
+    AccountID,
     RawProfile,
     RawAccountMigration,
     InvalidAccountAgentIDError,
@@ -82,7 +81,7 @@ export class LocalNode {
         initialAgentSecret?: AgentSecret;
     }): Promise<{
         node: LocalNode;
-        accountID: RawAccountID;
+        accountID: AccountID;
         accountSecret: AgentSecret;
         sessionID: SessionID;
     }> {
@@ -174,7 +173,7 @@ export class LocalNode {
         crypto,
         migration,
     }: {
-        accountID: RawAccountID;
+        accountID: AccountID;
         accountSecret: AgentSecret;
         sessionID: SessionID | undefined;
         peersToLoadFrom: Peer[];
@@ -471,7 +470,7 @@ export class LocalNode {
     }
 
     /** @internal */
-    expectProfileLoaded(id: RawAccountID, expectation?: string): RawProfile {
+    expectProfileLoaded(id: AccountID, expectation?: string): RawProfile {
         const account = this.expectCoValueLoaded(id, expectation);
         const profileID = expectGroup(account.getCurrentContent()).get(
             "profile",
@@ -534,7 +533,7 @@ export class LocalNode {
 
     /** @internal */
     resolveAccountAgent(
-        id: RawAccountID | AgentID,
+        id: AccountID | AgentID,
         expectation?: string,
     ): Result<AgentID, ResolveAccountAgentError> {
         if (isAgentID(id)) {
@@ -561,7 +560,7 @@ export class LocalNode {
     }
 
     resolveAccountAgentAsync(
-        id: RawAccountID | AgentID,
+        id: AccountID | AgentID,
         expectation?: string,
     ): ResultAsync<AgentID, ResolveAccountAgentError> {
         if (isAgentID(id)) {
@@ -607,12 +606,12 @@ export class LocalNode {
     /**
      * @deprecated use Account.createGroup() instead
      */
-    createGroup(uniqueness: CoValueUniqueness = this.crypto.createdNowUnique()): RawGroup {
+    createGroup(): RawGroup {
         const groupCoValue = this.createCoValue({
             type: "comap",
             ruleset: { type: "group", initialAdmin: this.account.id },
             meta: null,
-            ...uniqueness
+            ...this.crypto.createdNowUnique(),
         });
 
         const group = expectGroup(groupCoValue.getCurrentContent());
@@ -732,19 +731,19 @@ export type LoadCoValueCoreError = {
     type: "ErrorLoadingCoValueCore";
     error: unknown;
     expectation?: string;
-    id: RawAccountID;
+    id: AccountID;
 };
 
 export type AccountUnavailableFromAllPeersError = {
     type: "AccountUnavailableFromAllPeers";
     expectation?: string;
-    id: RawAccountID;
+    id: AccountID;
 };
 
 export type UnexpectedlyNotAccountError = {
     type: "UnexpectedlyNotAccount";
     expectation?: string;
-    id: RawAccountID;
+    id: AccountID;
 };
 
 export type ResolveAccountAgentError =

package/src/permissions.ts

@@ -5,13 +5,13 @@ import { KeyID } from "./crypto/crypto.js";
 import { CoValueCore, Transaction } from "./coValueCore.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
-import { RawAccount, RawAccountID, RawProfile } from "./coValues/account.js";
+import { RawAccount, AccountID, RawProfile } from "./coValues/account.js";
 import { parseJSON } from "./jsonStringify.js";
 import { EVERYONE, Everyone } from "./coValues/group.js";
 import { expectGroup } from "./typeUtils/expectGroup.js";
 
 export type PermissionsDef =
-    | { type: "group"; initialAdmin: RawAccountID | AgentID }
+    | { type: "group"; initialAdmin: AccountID | AgentID }
     | { type: "ownedByGroup"; group: RawCoID }
     | { type: "unsafeAllowAll" };
 
@@ -53,7 +53,7 @@ export function determineValidTransactions(
         }
 
         const memberState: {
-            [agent: RawAccountID | AgentID]: Role;
+            [agent: AccountID | AgentID]: Role;
             [EVERYONE]?: Role;
         } = {};
 
@@ -99,7 +99,7 @@ export function determineValidTransactions(
             }
 
             const change = changes[0] as
-                | MapOpPayload<RawAccountID | AgentID | Everyone, Role>
+                | MapOpPayload<AccountID | AgentID | Everyone, Role>
                 | MapOpPayload<"readKey", JsonValue>
                 | MapOpPayload<"profile", CoID<RawProfile>>;
             if (changes.length !== 1) {
@@ -303,7 +303,7 @@ export function isKeyForKeyField(co: string): co is `${KeyID}_for_${KeyID}` {
 
 export function isKeyForAccountField(
     co: string,
-): co is `${KeyID}_for_${RawAccountID | AgentID}` {
+): co is `${KeyID}_for_${AccountID | AgentID}` {
     return (
         (co.startsWith("key_") &&
             (co.includes("_for_sealer") || co.includes("_for_co"))) ||

package/src/priority.ts

@@ -0,0 +1,39 @@
+import { type CoValueHeader } from "./coValueCore.js";
+
+/**
+ * The priority of a `CoValue` determines how much priority is given
+ * to its content messages.
+ * 
+ * The priority value is handled as weight in the weighed round robin algorithm
+ * used to determine the order in which messages are sent.
+ * 
+ * Follows the HTTP urgency range and order:
+ *  - https://www.rfc-editor.org/rfc/rfc9218.html#name-urgency
+ */
+export const CO_VALUE_PRIORITY = {
+    HIGH: 0,
+    MEDIUM: 3,
+    LOW: 6,
+} as const;
+
+export type CoValuePriority = 0 | 1 | 2 | 3 | 4 | 5 | 6 | 7;
+
+export function getPriorityFromHeader(header: CoValueHeader | undefined | boolean): CoValuePriority {
+    if (typeof header === "boolean" || !header) {
+        return CO_VALUE_PRIORITY.MEDIUM;
+    }
+
+    if (header.meta?.type === "account") {
+        return CO_VALUE_PRIORITY.HIGH;
+    }
+
+    if (header.ruleset.type === "group") {
+        return CO_VALUE_PRIORITY.HIGH;
+    }
+
+    if (header.type === "costream" && header.meta?.type === "binary") {
+        return CO_VALUE_PRIORITY.LOW;
+    }
+
+    return CO_VALUE_PRIORITY.MEDIUM;
+}

package/src/storage/chunksAndKnownStates.ts

@@ -1,5 +1,6 @@
 import { RawCoID, SessionID } from "../ids.js";
 import { MAX_RECOMMENDED_TX_SIZE } from "../index.js";
+import { getPriorityFromHeader } from "../priority.js";
 import { CoValueKnownState, NewContentMessage } from "../sync.js";
 import { CoValueChunk } from "./index.js";
 
@@ -15,6 +16,7 @@ export function contentSinceChunk(
         action: "content",
         header: known?.header ? undefined : chunk.header,
         new: {},
+        priority: getPriorityFromHeader(chunk.header),
     });
 
     for (const [sessionID, sessionsEntry] of Object.entries(

package/src/sync.ts

@@ -3,6 +3,8 @@ import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
 import { LocalNode, newLoadingState } from "./localNode.js";
 import { RawCoID, SessionID } from "./ids.js";
+import { PeerState } from "./PeerState.js";
+import { CoValuePriority } from "./priority.js";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -38,6 +40,7 @@ export type NewContentMessage = {
     action: "content";
     id: RawCoID;
     header?: CoValueHeader;
+    priority: CoValuePriority;
     new: {
         [sessionID: SessionID]: SessionNewContent;
     };
@@ -76,17 +79,6 @@ export interface Peer {
     crashOnClose: boolean;
 }
 
-export interface PeerState {
-    id: PeerID;
-    optimisticKnownStates: { [id: RawCoID]: CoValueKnownState };
-    toldKnownState: Set<RawCoID>;
-    incoming: IncomingSyncStream;
-    outgoing: OutgoingSyncQueue;
-    role: "peer" | "server" | "client";
-    priority?: number;
-    crashOnClose: boolean;
-}
-
 export function combinedKnownStates(
     stateA: CoValueKnownState,
     stateB: CoValueKnownState,
@@ -141,7 +133,7 @@ export class SyncManager {
 
         for (const peer of eligiblePeers) {
             // console.log("loading", id, "from", peer.id);
-            await peer.outgoing.push({
+            await peer.pushOutgoingMessage({
                 action: "load",
                 id: id,
                 header: false,
@@ -227,7 +219,7 @@ export class SyncManager {
                 id,
                 header: false,
                 sessions: {},
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending load", e);
             });
             return;
@@ -244,7 +236,7 @@ export class SyncManager {
             this.trySendToPeer(peer, {
                 action: "load",
                 ...coValue.knownState(),
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending load", e);
             });
         }
@@ -274,7 +266,7 @@ export class SyncManager {
                 action: "known",
                 asDependencyOf,
                 ...coValue.knownState(),
-            }).catch((e) => {
+            }).catch((e: unknown) => {
                 console.error("Error sending known state", e);
             });
 
@@ -282,7 +274,10 @@ export class SyncManager {
         }
     }
 
-    async sendNewContentIncludingDependencies(id: RawCoID, peer: PeerState) {
+    async sendNewContentIncludingDependencies(
+        id: RawCoID,
+        peer: PeerState,
+    ) {
         const coValue = this.local.expectCoValueLoaded(id);
 
         await Promise.all(
@@ -310,9 +305,11 @@ export class SyncManager {
                     //     } header: ${!!piece.header}`,
                     //     // Object.values(piece.new).map((s) => s.newTransactions)
                     // );
-                    this.trySendToPeer(peer, piece).catch((e) => {
+
+                    this.trySendToPeer(peer, piece).catch((e: unknown) => {
                         console.error("Error sending content piece", e);
                     });
+
                     if (performance.now() - lastYield > 10) {
                         await new Promise<void>((resolve) => {
                             setTimeout(resolve, 0);
@@ -336,16 +333,7 @@ export class SyncManager {
     }
 
     addPeer(peer: Peer) {
-        const peerState: PeerState = {
-            id: peer.id,
-            optimisticKnownStates: {},
-            incoming: peer.incoming,
-            outgoing: peer.outgoing,
-            toldKnownState: new Set(),
-            role: peer.role,
-            priority: peer.priority,
-            crashOnClose: peer.crashOnClose,
-        };
+        const peerState = new PeerState(peer);
         this.peers[peer.id] = peerState;
 
         if (peer.role === "server") {
@@ -420,7 +408,7 @@ export class SyncManager {
     }
 
     trySendToPeer(peer: PeerState, msg: SyncMessage) {
-        return peer.outgoing.push(msg);
+        return peer.pushOutgoingMessage(msg);
     }
 
     async handleLoad(msg: LoadMessage, peer: PeerState) {
@@ -679,7 +667,8 @@ export class SyncManager {
                     newTransactions.length + " new transactions",
                     "after: " + newContentForSession.after,
                     "our last known tx idx initially: " + ourKnownTxIdx,
-                    "our last known tx idx now: " + coValue.sessionLogs.get(sessionID)?.transactions.length,
+                    "our last known tx idx now: " +
+                        coValue.sessionLogs.get(sessionID)?.transactions.length,
                 );
                 continue;
             }
@@ -774,11 +763,7 @@ export class SyncManager {
 
     gracefulShutdown() {
         for (const peer of Object.values(this.peers)) {
-            console.debug("Gracefully closing", peer.id);
-            peer.outgoing.close();
-            peer.incoming = (async function* () {
-                yield "Disconnected" as const;
-            })();
+            peer.gracefulShutdown();
         }
     }
 }