cojson 0.4.13 → 0.5.1

package/src/crypto.ts

@@ -1,4 +1,12 @@
-import { ed25519, x25519 } from "@noble/curves/ed25519";
+import {
+    initBundledOnce,
+    Ed25519SigningKey,
+    Ed25519VerifyingKey,
+    X25519StaticSecret,
+    Memory,
+    Ed25519Signature,
+    X25519PublicKey,
+} from "@hazae41/berith";
 import { xsalsa20_poly1305, xsalsa20 } from "@noble/ciphers/salsa";
 import { JsonValue } from "./jsonValue.js";
 import { base58 } from "@scure/base";
@@ -10,7 +18,13 @@ import { createBLAKE3 } from "hash-wasm";
 import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 
 let blake3Instance: Awaited<ReturnType<typeof createBLAKE3>>;
-let blake3HashOnce: (data: Uint8Array) => Uint8Array;
+let blake3HashOnce: (data: Uint8Array) => Uint8Array = () => {
+    throw new Error(
+        "cojson WASM dependencies not yet loaded; Make sure to import `cojsonReady` from `cojson` and await it before using any cojson functionality:\n\n" +
+            'import { cojsonReady } from "cojson";\n' +
+            "await cojsonReady;\n\n"
+    );
+};
 let blake3HashOnceWithContext: (
     data: Uint8Array,
     { context }: { context: Uint8Array }
@@ -21,29 +35,36 @@ let blake3incrementalUpdateSLOW_WITH_DEVTOOLS: (
 ) => Uint8Array;
 let blake3digestForState: (state: Uint8Array) => Uint8Array;
 
-export const cryptoReady = new Promise<void>((resolve) => {
-    createBLAKE3()
-        .then((bl3) => {
-            blake3Instance = bl3;
-            blake3HashOnce = (data) => {
-                return bl3.init().update(data).digest("binary");
-            };
-            blake3HashOnceWithContext = (data, { context }) => {
-                return bl3.init().update(context).update(data).digest("binary");
-            };
-            blake3incrementalUpdateSLOW_WITH_DEVTOOLS = (state, data) => {
-                bl3.load(state).update(data);
-                return bl3.save();
-            };
-            blake3digestForState = (state) => {
-                return bl3.load(state).digest("binary");
-            };
-            resolve();
-        })
-        .catch((e) =>
-            console.error("Failed to load cryptography dependencies", e)
-        );
-});
+export const cryptoReady = Promise.all([
+    new Promise<void>((resolve) => {
+        createBLAKE3()
+            .then((bl3) => {
+                blake3Instance = bl3;
+                blake3HashOnce = (data) => {
+                    return bl3.init().update(data).digest("binary");
+                };
+                blake3HashOnceWithContext = (data, { context }) => {
+                    return bl3
+                        .init()
+                        .update(context)
+                        .update(data)
+                        .digest("binary");
+                };
+                blake3incrementalUpdateSLOW_WITH_DEVTOOLS = (state, data) => {
+                    bl3.load(state).update(data);
+                    return bl3.save();
+                };
+                blake3digestForState = (state) => {
+                    return bl3.load(state).digest("binary");
+                };
+                resolve();
+            })
+            .catch((e) =>
+                console.error("Failed to load cryptography dependencies", e)
+            );
+    }),
+    initBundledOnce(),
+]);
 
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
@@ -59,7 +80,9 @@ const textEncoder = new TextEncoder();
 const textDecoder = new TextDecoder();
 
 export function newRandomSigner(): SignerSecret {
-    return `signerSecret_z${base58.encode(ed25519.utils.randomPrivateKey())}`;
+    return `signerSecret_z${base58.encode(
+        new Ed25519SigningKey().to_bytes().copyAndDispose()
+    )}`;
 }
 
 export function signerSecretToBytes(secret: SignerSecret): Uint8Array {
@@ -72,17 +95,22 @@ export function signerSecretFromBytes(bytes: Uint8Array): SignerSecret {
 
 export function getSignerID(secret: SignerSecret): SignerID {
     return `signer_z${base58.encode(
-        ed25519.getPublicKey(
-            base58.decode(secret.substring("signerSecret_z".length))
+        Ed25519SigningKey.from_bytes(
+            new Memory(base58.decode(secret.substring("signerSecret_z".length)))
         )
+            .public()
+            .to_bytes()
+            .copyAndDispose()
     )}`;
 }
 
 export function sign(secret: SignerSecret, message: JsonValue): Signature {
-    const signature = ed25519.sign(
-        textEncoder.encode(stableStringify(message)),
-        base58.decode(secret.substring("signerSecret_z".length))
-    );
+    const signature = Ed25519SigningKey.from_bytes(
+        new Memory(base58.decode(secret.substring("signerSecret_z".length)))
+    )
+        .sign(new Memory(textEncoder.encode(stableStringify(message))))
+        .to_bytes()
+        .copyAndDispose();
     return `signature_z${base58.encode(signature)}`;
 }
 
@@ -91,15 +119,20 @@ export function verify(
     message: JsonValue,
     id: SignerID
 ): boolean {
-    return ed25519.verify(
-        base58.decode(signature.substring("signature_z".length)),
-        textEncoder.encode(stableStringify(message)),
-        base58.decode(id.substring("signer_z".length))
+    return new Ed25519VerifyingKey(
+        new Memory(base58.decode(id.substring("signer_z".length)))
+    ).verify(
+        new Memory(textEncoder.encode(stableStringify(message))),
+        new Ed25519Signature(
+            new Memory(base58.decode(signature.substring("signature_z".length)))
+        )
     );
 }
 
 export function newRandomSealer(): SealerSecret {
-    return `sealerSecret_z${base58.encode(x25519.utils.randomPrivateKey())}`;
+    return `sealerSecret_z${base58.encode(
+        new X25519StaticSecret().to_bytes().copyAndDispose()
+    )}`;
 }
 
 export function sealerSecretToBytes(secret: SealerSecret): Uint8Array {
@@ -112,9 +145,12 @@ export function sealerSecretFromBytes(bytes: Uint8Array): SealerSecret {
 
 export function getSealerID(secret: SealerSecret): SealerID {
     return `sealer_z${base58.encode(
-        x25519.getPublicKey(
-            base58.decode(secret.substring("sealerSecret_z".length))
+        X25519StaticSecret.from_bytes(
+            new Memory(base58.decode(secret.substring("sealerSecret_z".length)))
         )
+            .to_public()
+            .to_bytes()
+            .copyAndDispose()
     )}`;
 }
 
@@ -180,7 +216,10 @@ export function seal<T extends JsonValue>({
 
     const plaintext = textEncoder.encode(stableStringify(message));
 
-    const sharedSecret = x25519.getSharedSecret(senderPriv, sealerPub);
+    const sharedSecret = X25519StaticSecret.from_bytes(new Memory(senderPriv))
+        .diffie_hellman(X25519PublicKey.from_bytes(new Memory(sealerPub)))
+        .to_bytes()
+        .copyAndDispose();
 
     const sealedBytes = xsalsa20_poly1305(sharedSecret, nOnce).encrypt(
         plaintext
@@ -205,7 +244,10 @@ export function unseal<T extends JsonValue>(
 
     const sealedBytes = base64URLtoBytes(sealed.substring("sealed_U".length));
 
-    const sharedSecret = x25519.getSharedSecret(sealerPriv, senderPub);
+    const sharedSecret = X25519StaticSecret.from_bytes(new Memory(sealerPriv))
+        .diffie_hellman(X25519PublicKey.from_bytes(new Memory(senderPub)))
+        .to_bytes()
+        .copyAndDispose();
 
     const plaintext = xsalsa20_poly1305(sharedSecret, nOnce).decrypt(
         sealedBytes

package/src/localNode.ts

@@ -19,7 +19,7 @@ import {
     Group,
     secretSeedFromInviteSecret,
 } from "./coValues/group.js";
-import { Peer, SyncManager } from "./sync.js";
+import { Peer, PeerID, SyncManager } from "./sync.js";
 import { AgentID, RawCoID, SessionID, isAgentID } from "./ids.js";
 import { CoID } from "./coValue.js";
 import {
@@ -109,12 +109,26 @@ export class LocalNode {
 
         if (migration) {
             migration(accountOnNodeWithAccount, profile as P);
-            nodeWithAccount.account = new ControlledAccount(
-                accountOnNodeWithAccount.core,
-                accountOnNodeWithAccount.agentSecret
-            );
         }
 
+        nodeWithAccount.account = new ControlledAccount(
+            accountOnNodeWithAccount.core,
+            accountOnNodeWithAccount.agentSecret
+        );
+
+        // we shouldn't need this, but it fixes account data not syncing for new accounts
+        function syncAllCoValuesAfterCreateAccount() {
+            for (const coValueEntry of Object.values(nodeWithAccount.coValues)) {
+                if (coValueEntry.state === "loaded") {
+                    void nodeWithAccount.syncManager.syncCoValue(coValueEntry.coValue);
+                }
+            }
+        }
+
+        syncAllCoValuesAfterCreateAccount();
+
+        setTimeout(syncAllCoValuesAfterCreateAccount, 500);
+
         return {
             node: nodeWithAccount,
             accountID: accountOnNodeWithAccount.id,
@@ -153,6 +167,11 @@ export class LocalNode {
         }
 
         const account = await accountPromise;
+
+        if (account === "unavailable") {
+            throw new Error("Account unavailable from all peers");
+        }
+
         const controlledAccount = new ControlledAccount(
             account.core,
             accountSecret
@@ -199,14 +218,36 @@ export class LocalNode {
     }
 
     /** @internal */
-    loadCoValue(id: RawCoID, onProgress?: (progress: number) => void): Promise<CoValueCore> {
+    async loadCoValueCore(
+        id: RawCoID,
+        options: {
+            dontLoadFrom?: PeerID;
+            dontWaitFor?: PeerID;
+            onProgress?: (progress: number) => void;
+        } = {}
+    ): Promise<CoValueCore | "unavailable"> {
         let entry = this.coValues[id];
         if (!entry) {
-            entry = newLoadingState(onProgress);
+            const peersToWaitFor = new Set(
+                Object.values(this.syncManager.peers)
+                    .filter((peer) => peer.role === "server")
+                    .map((peer) => peer.id)
+            );
+            if (options.dontWaitFor) peersToWaitFor.delete(options.dontWaitFor);
+            entry = newLoadingState(peersToWaitFor, options.onProgress);
 
             this.coValues[id] = entry;
 
-            this.syncManager.loadFromPeers(id);
+            this.syncManager
+                .loadFromPeers(id, options.dontLoadFrom)
+                .catch((e) => {
+                    console.error(
+                        "Error loading from peers",
+                        id,
+
+                        e
+                    );
+                });
         }
         if (entry.state === "loaded") {
             return Promise.resolve(entry.coValue);
@@ -221,25 +262,38 @@ export class LocalNode {
      *
      * @category 3. Low-level
      */
-    async load<T extends CoValue>(id: CoID<T>, onProgress?: (progress: number) => void): Promise<T> {
-        return (await this.loadCoValue(id, onProgress)).getCurrentContent() as T;
+    async load<T extends CoValue>(
+        id: CoID<T>,
+        onProgress?: (progress: number) => void
+    ): Promise<T | "unavailable"> {
+        const core = await this.loadCoValueCore(id, { onProgress });
+
+        if (core === "unavailable") {
+            return "unavailable";
+        }
+
+        return core.getCurrentContent() as T;
     }
 
     /** @category 3. Low-level */
     subscribe<T extends CoValue>(
         id: CoID<T>,
-        callback: (update: T) => void
+        callback: (update: T | "unavailable") => void
     ): () => void {
         let stopped = false;
         let unsubscribe!: () => void;
 
-        console.log("Subscribing to " + id);
+        // console.log("Subscribing to " + id);
 
         this.load(id)
             .then((coValue) => {
                 if (stopped) {
                     return;
                 }
+                if (coValue === "unavailable") {
+                    callback("unavailable");
+                    return;
+                }
                 unsubscribe = coValue.subscribe(callback);
             })
             .catch((e) => {
@@ -260,6 +314,12 @@ export class LocalNode {
     ): Promise<void> {
         const groupOrOwnedValue = await this.load(groupOrOwnedValueID);
 
+        if (groupOrOwnedValue === "unavailable") {
+            throw new Error(
+                "Trying to accept invite: Group/owned value unavailable from all peers"
+            );
+        }
+
         if (groupOrOwnedValue.core.header.ruleset.type === "ownedByGroup") {
             return this.acceptInvite(
                 groupOrOwnedValue.core.header.ruleset.group as CoID<Group>,
@@ -325,7 +385,7 @@ export class LocalNode {
                 : "reader"
         );
 
-        group.core._sessions = groupAsInvite.core.sessions;
+        group.core._sessionLogs = groupAsInvite.core.sessionLogs;
         group.core._cachedContent = undefined;
 
         for (const groupListener of group.core.listeners) {
@@ -400,17 +460,6 @@ export class LocalNode {
                 },
             });
 
-            console.log(
-                "Creating read key",
-                getAgentSealerSecret(agentSecret),
-                getAgentSealerID(accountAgentID),
-                account.id,
-                account.core.nextTransactionID(),
-                "in session",
-                account.core.node.currentSessionID,
-                "=",
-                sealed
-            );
             editable.set(
                 `${readKey.id}_for_${accountAgentID}`,
                 sealed,
@@ -432,16 +481,13 @@ export class LocalNode {
 
         const accountOnThisNode = this.expectCoValueLoaded(account.id);
 
-        accountOnThisNode._sessions = {
-            ...account.core.sessions,
-        };
+        accountOnThisNode._sessionLogs = new Map(account.core.sessionLogs);
+
         accountOnThisNode._cachedContent = undefined;
 
         const profileOnThisNode = this.createCoValue(profile.core.header);
 
-        profileOnThisNode._sessions = {
-            ...profile.core.sessions,
-        };
+        profileOnThisNode._sessionLogs = new Map(profile.core.sessionLogs);
         profileOnThisNode._cachedContent = undefined;
 
         return new ControlledAccount(accountOnThisNode, agentSecret);
@@ -475,6 +521,41 @@ export class LocalNode {
         return new Account(coValue).getCurrentAgentID();
     }
 
+    async resolveAccountAgentAsync(
+        id: AccountID | AgentID,
+        expectation?: string
+    ): Promise<AgentID> {
+        if (isAgentID(id)) {
+            return id;
+        }
+
+        const coValue = await this.loadCoValueCore(id);
+
+        if (coValue === "unavailable") {
+            throw new Error(
+                `${
+                    expectation ? expectation + ": " : ""
+                }Account ${id} is unavailable from all peers`
+            );
+        }
+
+        if (
+            coValue.header.type !== "comap" ||
+            coValue.header.ruleset.type !== "group" ||
+            !coValue.header.meta ||
+            !("type" in coValue.header.meta) ||
+            coValue.header.meta.type !== "account"
+        ) {
+            throw new Error(
+                `${
+                    expectation ? expectation + ": " : ""
+                }CoValue ${id} is not an account`
+            );
+        }
+
+        return new Account(coValue).getCurrentAgentID();
+    }
+
     /**
      * @deprecated use Account.createGroup() instead
      */
@@ -543,7 +624,7 @@ export class LocalNode {
                 const newCoValue = new CoValueCore(
                     entry.coValue.header,
                     newNode,
-                    { ...entry.coValue.sessions }
+                    new Map(entry.coValue.sessionLogs)
                 );
 
                 newNode.coValues[coValueID as RawCoID] = {
@@ -575,17 +656,34 @@ export class LocalNode {
 type CoValueState =
     | {
           state: "loading";
-          done: Promise<CoValueCore>;
-          resolve: (coValue: CoValueCore) => void;
+          done: Promise<CoValueCore | "unavailable">;
+          resolve: (coValue: CoValueCore | "unavailable") => void;
           onProgress?: (progress: number) => void;
+          firstPeerState: {
+              [peerID: string]:
+                  | {
+                        type: "waiting";
+                        done: Promise<void>;
+                        resolve: () => void;
+                    }
+                  | { type: "available" }
+                  | { type: "unavailable" };
+          };
       }
-    | { state: "loaded"; coValue: CoValueCore; onProgress?: (progress: number) => void; };
+    | {
+          state: "loaded";
+          coValue: CoValueCore;
+          onProgress?: (progress: number) => void;
+      };
 
 /** @internal */
-export function newLoadingState(onProgress?: (progress: number) => void): CoValueState {
-    let resolve: (coValue: CoValueCore) => void;
+export function newLoadingState(
+    currentPeerIds: Set<PeerID>,
+    onProgress?: (progress: number) => void
+): CoValueState {
+    let resolve: (coValue: CoValueCore | "unavailable") => void;
 
-    const promise = new Promise<CoValueCore>((r) => {
+    const promise = new Promise<CoValueCore | "unavailable">((r) => {
         resolve = r;
     });
 
@@ -593,6 +691,15 @@ export function newLoadingState(onProgress?: (progress: number) => void): CoValu
         state: "loading",
         done: promise,
         resolve: resolve!,
-        onProgress
+        onProgress,
+        firstPeerState: Object.fromEntries(
+            [...currentPeerIds].map((id) => {
+                let resolve: () => void;
+                const done = new Promise<void>((r) => {
+                    resolve = r;
+                });
+                return [id, { type: "waiting", done, resolve: resolve! }];
+            })
+        ),
     };
 }

package/src/permissions.ts

@@ -2,10 +2,7 @@ import { CoID } from "./coValue.js";
 import { MapOpPayload } from "./coValues/coMap.js";
 import { JsonValue } from "./jsonValue.js";
 import { KeyID } from "./crypto.js";
-import {
-    CoValueCore,
-    Transaction,
-} from "./coValueCore.js";
+import { CoValueCore, Transaction } from "./coValueCore.js";
 import { accountOrAgentIDfromSessionID } from "./typeUtils/accountOrAgentIDfromSessionID.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
 import { Account, AccountID, Profile } from "./coValues/account.js";
@@ -31,19 +28,19 @@ export function determineValidTransactions(
     coValue: CoValueCore
 ): { txID: TransactionID; tx: Transaction }[] {
     if (coValue.header.ruleset.type === "group") {
-        const allTransactionsSorted = Object.entries(coValue.sessions).flatMap(
-            ([sessionID, sessionLog]) => {
-                return sessionLog.transactions.map((tx, txIndex) => ({
-                    sessionID,
-                    txIndex,
-                    tx,
-                })) as {
-                    sessionID: SessionID;
-                    txIndex: number;
-                    tx: Transaction;
-                }[];
-            }
-        );
+        const allTransactionsSorted = [
+            ...coValue.sessionLogs.entries(),
+        ].flatMap(([sessionID, sessionLog]) => {
+            return sessionLog.transactions.map((tx, txIndex) => ({
+                sessionID,
+                txIndex,
+                tx,
+            })) as {
+                sessionID: SessionID;
+                txIndex: number;
+                tx: Transaction;
+            }[];
+        });
 
         allTransactionsSorted.sort((a, b) => {
             return a.tx.madeAt - b.tx.madeAt;
@@ -242,11 +239,9 @@ export function determineValidTransactions(
             throw new Error("Group must be a map");
         }
 
-        return Object.entries(coValue.sessions).flatMap(
+        return [...coValue.sessionLogs.entries()].flatMap(
             ([sessionID, sessionLog]) => {
-                const transactor = accountOrAgentIDfromSessionID(
-                    sessionID as SessionID
-                );
+                const transactor = accountOrAgentIDfromSessionID(sessionID);
 
                 return sessionLog.transactions
                     .filter((tx) => {
@@ -266,16 +261,16 @@ export function determineValidTransactions(
                         );
                     })
                     .map((tx, txIndex) => ({
-                        txID: { sessionID: sessionID as SessionID, txIndex },
+                        txID: { sessionID: sessionID, txIndex },
                         tx,
                     }));
             }
         );
     } else if (coValue.header.ruleset.type === "unsafeAllowAll") {
-        return Object.entries(coValue.sessions).flatMap(
+        return [...coValue.sessionLogs.entries()].flatMap(
             ([sessionID, sessionLog]) => {
                 return sessionLog.transactions.map((tx, txIndex) => ({
-                    txID: { sessionID: sessionID as SessionID, txIndex },
+                    txID: { sessionID: sessionID, txIndex },
                     tx,
                 }));
             }

package/src/streamUtils.ts

@@ -18,11 +18,11 @@ export function connectedPeers(
         peer2role?: Peer["role"];
     } = {}
 ): [Peer, Peer] {
-    const [inRx1, inTx1] = newStreamPair<SyncMessage>();
-    const [outRx1, outTx1] = newStreamPair<SyncMessage>();
+    const [inRx1, inTx1] = newStreamPair<SyncMessage>(peer1id + "_in");
+    const [outRx1, outTx1] = newStreamPair<SyncMessage>(peer1id + "_out");
 
-    const [inRx2, inTx2] = newStreamPair<SyncMessage>();
-    const [outRx2, outTx2] = newStreamPair<SyncMessage>();
+    const [inRx2, inTx2] = newStreamPair<SyncMessage>(peer2id + "_in");
+    const [outRx2, outTx2] = newStreamPair<SyncMessage>(peer2id + "_out");
 
     void outRx2
         .pipeThrough(
@@ -37,7 +37,7 @@ export function connectedPeers(
                             JSON.stringify(
                                 chunk,
                                 (k, v) =>
-                                    (k === "changes" || k === "encryptedChanges")
+                                    k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
                                 2
@@ -62,7 +62,7 @@ export function connectedPeers(
                             JSON.stringify(
                                 chunk,
                                 (k, v) =>
-                                    (k === "changes" || k === "encryptedChanges")
+                                    k === "changes" || k === "encryptedChanges"
                                         ? v.slice(0, 20) + "..."
                                         : v,
                                 2
@@ -91,7 +91,10 @@ export function connectedPeers(
     return [peer1AsPeer, peer2AsPeer];
 }
 
-export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
+export function newStreamPair<T>(
+    pairName?: string
+): [ReadableStream<T>, WritableStream<T>] {
+    let queueLength = 0;
     let readerClosed = false;
 
     let resolveEnqueue: (enqueue: (item: T) => void) => void;
@@ -104,6 +107,22 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
         resolveClose = resolve;
     });
 
+    let queueWasOverflowing = false;
+
+    function maybeReportQueueLength() {
+        if (queueLength >= 100) {
+            queueWasOverflowing = true;
+            if (queueLength % 100 === 0) {
+                console.warn(pairName, "overflowing queue length", queueLength);
+            }
+        } else {
+            if (queueWasOverflowing) {
+                console.debug(pairName, "ok queue length", queueLength);
+                queueWasOverflowing = false;
+            }
+        }
+    }
+
     const readable = new ReadableStream<T>({
         async start(controller) {
             resolveEnqueue(controller.enqueue.bind(controller));
@@ -114,12 +133,26 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
             console.log("Manually closing reader");
             readerClosed = true;
         },
-    });
+    }).pipeThrough(
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        new TransformStream<any, any>({
+            transform(
+                chunk: SyncMessage,
+                controller: { enqueue: (msg: SyncMessage) => void }
+            ) {
+                queueLength -= 1;
+                maybeReportQueueLength();
+                controller.enqueue(chunk);
+            },
+        })
+    ) as ReadableStream<T>;
 
     let lastWritePromise = Promise.resolve();
 
     const writable = new WritableStream<T>({
         async write(chunk) {
+            queueLength += 1;
+            maybeReportQueueLength();
             const enqueue = await enqueuePromise;
             if (readerClosed) {
                 throw new Error("Reader closed");