cojson 0.1.10 → 0.1.12

package/src/coValueCore.ts

@@ -100,6 +100,7 @@ export class CoValueCore {
     _sessions: { [key: SessionID]: SessionLog };
     _cachedContent?: CoValueImpl;
     listeners: Set<(content?: CoValueImpl) => void> = new Set();
+    _decryptionCache: {[key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined} = {}
 
     constructor(
         header: CoValueHeader,
@@ -186,10 +187,16 @@ export class CoValueCore {
             return false;
         }
 
+        // const beforeHash = performance.now();
         const { expectedNewHash, newStreamingHash } = this.expectedNewHashAfter(
             sessionID,
             newTransactions
         );
+        // const afterHash = performance.now();
+        // console.log(
+        //     "Hashing took",
+        //     afterHash - beforeHash
+        // );
 
         if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
             console.warn("Invalid hash", {
@@ -199,6 +206,7 @@ export class CoValueCore {
             return false;
         }
 
+        // const beforeVerify = performance.now();
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
                 "Invalid signature",
@@ -208,6 +216,11 @@ export class CoValueCore {
             );
             return false;
         }
+        // const afterVerify = performance.now();
+        // console.log(
+        //     "Verify took",
+        //     afterVerify - beforeVerify
+        // );
 
         const transactions = this.sessions[sessionID]?.transactions ?? [];
 
@@ -222,10 +235,105 @@ export class CoValueCore {
 
         this._cachedContent = undefined;
 
-        const content = this.getCurrentContent();
+        if (this.listeners.size > 0) {
+            const content = this.getCurrentContent();
+            for (const listener of this.listeners) {
+                listener(content);
+            }
+        }
+
+        return true;
+    }
 
-        for (const listener of this.listeners) {
-            listener(content);
+    async tryAddTransactionsAsync(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        givenExpectedNewHash: Hash | undefined,
+        newSignature: Signature
+    ): Promise<boolean> {
+        const signerID = getAgentSignerID(
+            this.node.resolveAccountAgent(
+                accountOrAgentIDfromSessionID(sessionID),
+                "Expected to know signer of transaction"
+            )
+        );
+
+        if (!signerID) {
+            console.warn(
+                "Unknown agent",
+                accountOrAgentIDfromSessionID(sessionID)
+            );
+            return false;
+        }
+
+        const nTxBefore = this.sessions[sessionID]?.transactions.length ?? 0;
+
+        // const beforeHash = performance.now();
+        const { expectedNewHash, newStreamingHash } = await this.expectedNewHashAfterAsync(
+            sessionID,
+            newTransactions
+        );
+        // const afterHash = performance.now();
+        // console.log(
+        //     "Hashing took",
+        //     afterHash - beforeHash
+        // );
+
+        const nTxAfter = this.sessions[sessionID]?.transactions.length ?? 0;
+
+        if (nTxAfter !== nTxBefore) {
+            const newTransactionLengthBefore = newTransactions.length;
+            newTransactions = newTransactions.slice((nTxAfter - nTxBefore));
+            console.warn("Transactions changed while async hashing", {
+                nTxBefore,
+                nTxAfter,
+                newTransactionLengthBefore,
+                remainingNewTransactions: newTransactions.length,
+            });
+        }
+
+        if (givenExpectedNewHash && givenExpectedNewHash !== expectedNewHash) {
+            console.warn("Invalid hash", {
+                expectedNewHash,
+                givenExpectedNewHash,
+            });
+            return false;
+        }
+
+        // const beforeVerify = performance.now();
+        if (!verify(newSignature, expectedNewHash, signerID)) {
+            console.warn(
+                "Invalid signature",
+                newSignature,
+                expectedNewHash,
+                signerID
+            );
+            return false;
+        }
+        // const afterVerify = performance.now();
+        // console.log(
+        //     "Verify took",
+        //     afterVerify - beforeVerify
+        // );
+
+        const transactions = this.sessions[sessionID]?.transactions ?? [];
+
+        transactions.push(...newTransactions);
+
+        this._sessions[sessionID] = {
+            transactions,
+            lastHash: expectedNewHash,
+            streamingHash: newStreamingHash,
+            lastSignature: newSignature,
+        };
+
+        this._cachedContent = undefined;
+
+        if (this.listeners.size > 0) {
+            const content = this.getCurrentContent();
+            for (const listener of this.listeners) {
+                listener(content);
+            }
         }
 
         return true;
@@ -259,6 +367,32 @@ export class CoValueCore {
         };
     }
 
+    async expectedNewHashAfterAsync(
+        sessionID: SessionID,
+        newTransactions: Transaction[]
+    ): Promise<{ expectedNewHash: Hash; newStreamingHash: StreamingHash }> {
+        const streamingHash =
+            this.sessions[sessionID]?.streamingHash.clone() ??
+            new StreamingHash();
+        let before = performance.now();
+        for (const transaction of newTransactions) {
+            streamingHash.update(transaction)
+            const after = performance.now();
+            if (after - before > 1) {
+                console.log("Hashing blocked for", after - before);
+                await new Promise((resolve) => setTimeout(resolve, 0));
+                before = performance.now();
+            }
+        }
+
+        const newStreamingHash = streamingHash.clone();
+
+        return {
+            expectedNewHash: streamingHash.digest(),
+            newStreamingHash,
+        };
+    }
+
     makeTransaction(
         changes: JsonValue[],
         privacy: "private" | "trusting"
@@ -276,14 +410,18 @@ export class CoValueCore {
                 );
             }
 
+            const encrypted = encryptForTransaction(changes, keySecret, {
+                in: this.id,
+                tx: this.nextTransactionID(),
+            });
+
+            this._decryptionCache[encrypted] = changes;
+
             transaction = {
                 privacy: "private",
                 madeAt,
                 keyUsed: keyID,
-                encryptedChanges: encryptForTransaction(changes, keySecret, {
-                    in: this.id,
-                    tx: this.nextTransactionID(),
-                }),
+                encryptedChanges: encrypted,
             };
         } else {
             transaction = {
@@ -359,14 +497,19 @@ export class CoValueCore {
                     if (!readKey) {
                         return undefined;
                     } else {
-                        const decrytedChanges = decryptForTransaction(
-                            tx.encryptedChanges,
-                            readKey,
-                            {
-                                in: this.id,
-                                tx: txID,
-                            }
-                        );
+                        let decrytedChanges = this._decryptionCache[tx.encryptedChanges];
+
+                        if (!decrytedChanges) {
+                            decrytedChanges = decryptForTransaction(
+                                tx.encryptedChanges,
+                                readKey,
+                                {
+                                    in: this.id,
+                                    tx: txID,
+                                }
+                            );
+                            this._decryptionCache[tx.encryptedChanges] = decrytedChanges;
+                        }
 
                         if (!decrytedChanges) {
                             console.error(

package/src/coValues/coStream.ts

@@ -3,7 +3,7 @@ import { CoID, ReadableCoValue, WriteableCoValue } from "../coValue.js";
 import { CoValueCore } from "../coValueCore.js";
 import { Group } from "../group.js";
 import { SessionID } from "../ids.js";
-import { base64url } from "@scure/base";
+import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
 
 export type BinaryChunkInfo = {
     mimeType: string;
@@ -17,7 +17,7 @@ export type BinaryStreamStart = {
 
 export type BinaryStreamChunk = {
     type: "chunk";
-    chunk: `U${string}`;
+    chunk: `binary_U${string}`;
 };
 
 export type BinaryStreamEnd = {
@@ -47,6 +47,7 @@ export class CoStream<
         this.id = core.id as CoID<CoStream<T, Meta>>;
         this.core = core;
         this.items = {};
+        this.fillFromCoValue();
     }
 
     get meta(): Meta {
@@ -110,6 +111,8 @@ export class CoStream<
     }
 }
 
+const binary_U_prefixLength = 8; // "binary_U".length;
+
 export class BinaryCoStream<
         Meta extends BinaryCoStreamMeta = { type: "binary" }
     >
@@ -121,6 +124,7 @@ export class BinaryCoStream<
     getBinaryChunks():
         | (BinaryChunkInfo & { chunks: Uint8Array[]; finished: boolean })
         | undefined {
+        const before = performance.now();
         const items = this.getSingleStream();
 
         if (!items) return;
@@ -134,15 +138,13 @@ export class BinaryCoStream<
 
         const chunks: Uint8Array[] = [];
 
+        let finished = false;
+        let totalLength = 0;
+
         for (const item of items.slice(1)) {
             if (item.type === "end") {
-                return {
-                    mimeType: start.mimeType,
-                    fileName: start.fileName,
-                    totalSizeBytes: start.totalSizeBytes,
-                    chunks,
-                    finished: true,
-                };
+                finished = true;
+                break;
             }
 
             if (item.type !== "chunk") {
@@ -150,15 +152,25 @@ export class BinaryCoStream<
                 return undefined;
             }
 
-            chunks.push(base64url.decode(item.chunk.slice(1)));
+            const chunk = base64URLtoBytes(
+                item.chunk.slice(binary_U_prefixLength)
+            );
+            totalLength += chunk.length;
+            chunks.push(chunk);
         }
 
+        const after = performance.now();
+        console.log(
+            "getBinaryChunks bandwidth in MB/s",
+            (1000 * totalLength) / (after - before) / (1024 * 1024)
+        );
+
         return {
             mimeType: start.mimeType,
             fileName: start.fileName,
             totalSizeBytes: start.totalSizeBytes,
             chunks,
-            finished: false,
+            finished,
         };
     }
 
@@ -205,10 +217,7 @@ export class WriteableBinaryCoStream<
     }
 
     /** @internal */
-    push(
-        item: BinaryStreamItem,
-        privacy: "private" | "trusting" = "private"
-    ) {
+    push(item: BinaryStreamItem, privacy: "private" | "trusting" = "private") {
         WriteableCoStream.prototype.push.call(this, item, privacy);
     }
 
@@ -229,13 +238,19 @@ export class WriteableBinaryCoStream<
         chunk: Uint8Array,
         privacy: "private" | "trusting" = "private"
     ) {
+        const before = performance.now();
         this.push(
             {
                 type: "chunk",
-                chunk: `U${base64url.encode(chunk)}`,
+                chunk: `binary_U${bytesToBase64url(chunk)}`,
             } satisfies BinaryStreamChunk,
             privacy
         );
+        const after = performance.now();
+        console.log(
+            "pushBinaryStreamChunk bandwidth in MB/s",
+            (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        );
     }
 
     endBinaryStream(privacy: "private" | "trusting" = "private") {

package/src/crypto.test.ts

@@ -21,6 +21,11 @@ import { xsalsa20_poly1305 } from "@noble/ciphers/salsa";
 import { blake3 } from "@noble/hashes/blake3";
 import stableStringify from "fast-json-stable-stringify";
 import { SessionID } from './ids.js';
+import { cojsonReady } from './index.js';
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Signatures round-trip and use stable stringify", () => {
     const data = { b: "world", a: "hello" };

package/src/crypto.ts

@@ -1,11 +1,39 @@
 import { ed25519, x25519 } from "@noble/curves/ed25519";
 import { xsalsa20_poly1305, xsalsa20 } from "@noble/ciphers/salsa";
 import { JsonValue } from "./jsonValue.js";
-import { base58, base64url } from "@scure/base";
-import stableStringify from "fast-json-stable-stringify";
-import { blake3 } from "@noble/hashes/blake3";
+import { base58 } from "@scure/base";
 import { randomBytes } from "@noble/ciphers/webcrypto/utils";
 import { AgentID, RawCoID, TransactionID } from "./ids.js";
+import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
+
+import { createBLAKE3 } from 'hash-wasm';
+import { stableStringify } from "./fastJsonStableStringify.js";
+
+let blake3Instance: Awaited<ReturnType<typeof createBLAKE3>>;
+let blake3HashOnce: (data: Uint8Array) => Uint8Array;
+let blake3HashOnceWithContext: (data: Uint8Array, {context}: {context: Uint8Array}) => Uint8Array;
+let blake3incrementalUpdateSLOW_WITH_DEVTOOLS: (state: Uint8Array, data: Uint8Array) => Uint8Array;
+let blake3digestForState: (state: Uint8Array) => Uint8Array;
+
+export const cryptoReady = new Promise<void>((resolve) => {
+    createBLAKE3().then(bl3 => {
+        blake3Instance = bl3;
+        blake3HashOnce = (data) => {
+            return bl3.init().update(data).digest('binary');
+        }
+        blake3HashOnceWithContext = (data, {context}) => {
+            return bl3.init().update(context).update(data).digest('binary');
+        }
+        blake3incrementalUpdateSLOW_WITH_DEVTOOLS = (state, data) => {
+            bl3.load(state).update(data);
+            return bl3.save();
+        }
+        blake3digestForState = (state) => {
+            return bl3.load(state).digest('binary');
+        }
+        resolve();
+    })
+});
 
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
@@ -127,7 +155,7 @@ export function seal<T extends JsonValue>(
     to: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): Sealed<T> {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -143,7 +171,7 @@ export function seal<T extends JsonValue>(
         plaintext
     );
 
-    return `sealed_U${base64url.encode(sealedBytes)}` as Sealed<T>;
+    return `sealed_U${bytesToBase64url(sealedBytes)}` as Sealed<T>;
 }
 
 export function unseal<T extends JsonValue>(
@@ -152,7 +180,7 @@ export function unseal<T extends JsonValue>(
     from: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): T | undefined {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -160,7 +188,7 @@ export function unseal<T extends JsonValue>(
 
     const senderPub = base58.decode(from.substring("sealer_z".length));
 
-    const sealedBytes = base64url.decode(sealed.substring("sealed_U".length));
+    const sealedBytes = base64URLtoBytes(sealed.substring("sealed_U".length));
 
     const sharedSecret = x25519.getSharedSecret(sealerPriv, senderPub);
 
@@ -180,28 +208,32 @@ export type Hash = `hash_z${string}`;
 
 export function secureHash(value: JsonValue): Hash {
     return `hash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value)))
+        blake3HashOnce(textEncoder.encode(stableStringify(value)))
     )}`;
 }
 
 export class StreamingHash {
-    state: ReturnType<typeof blake3.create>;
+    state: Uint8Array;
 
-    constructor(fromClone?: ReturnType<typeof blake3.create>) {
-        this.state = fromClone || blake3.create({});
+    constructor(fromClone?: Uint8Array) {
+        this.state = fromClone || blake3Instance.init().save();
     }
 
     update(value: JsonValue) {
-        this.state.update(textEncoder.encode(stableStringify(value)));
+        const encoded = textEncoder.encode(stableStringify(value))
+        // const before = performance.now();
+        this.state = blake3incrementalUpdateSLOW_WITH_DEVTOOLS(this.state, encoded);
+        // const after = performance.now();
+        // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
     }
 
     digest(): Hash {
-        const hash = this.state.digest();
+        const hash = blake3digestForState(this.state);
         return `hash_z${base58.encode(hash)}`;
     }
 
     clone(): StreamingHash {
-        return new StreamingHash(this.state.clone());
+        return new StreamingHash(new Uint8Array(this.state));
     }
 }
 
@@ -210,7 +242,10 @@ export const shortHashLength = 19;
 
 export function shortHash(value: JsonValue): ShortHash {
     return `shortHash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value))).slice(0, shortHashLength)
+        blake3HashOnce(textEncoder.encode(stableStringify(value))).slice(
+            0,
+            shortHashLength
+        )
     )}`;
 }
 
@@ -237,13 +272,13 @@ function encrypt<T extends JsonValue, N extends JsonValue>(
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
     const plaintext = textEncoder.encode(stableStringify(value));
     const ciphertext = xsalsa20(keySecretBytes, nOnce, plaintext);
-    return `encrypted_U${base64url.encode(ciphertext)}` as Encrypted<T, N>;
+    return `encrypted_U${bytesToBase64url(ciphertext)}` as Encrypted<T, N>;
 }
 
 export function encryptForTransaction<T extends JsonValue>(
@@ -289,11 +324,11 @@ function decrypt<T extends JsonValue, N extends JsonValue>(
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
-    const ciphertext = base64url.decode(
+    const ciphertext = base64URLtoBytes(
         encrypted.substring("encrypted_U".length)
     );
     const plaintext = xsalsa20(keySecretBytes, nOnce, ciphertext);
@@ -355,15 +390,17 @@ export function newRandomSecretSeed(): Uint8Array {
 
 export function agentSecretFromSecretSeed(secretSeed: Uint8Array): AgentSecret {
     if (secretSeed.length !== secretSeedLength) {
-        throw new Error(`Secret seed needs to be ${secretSeedLength} bytes long`);
+        throw new Error(
+            `Secret seed needs to be ${secretSeedLength} bytes long`
+        );
     }
 
     return `sealerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("seal"),
         })
     )}/signerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("sign"),
         })
     )}`;

package/src/fastJsonStableStringify.ts

@@ -0,0 +1,54 @@
+// adapted from fast-json-stable-stringify (https://github.com/epoberezkin/fast-json-stable-stringify)
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function stableStringify(data: any): string | undefined {
+    const cycles = false;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const seen: any[] = [];
+    let node = data;
+
+    if (node && node.toJSON && typeof node.toJSON === "function") {
+        node = node.toJSON();
+    }
+
+    if (node === undefined) return;
+    if (typeof node == "number") return isFinite(node) ? "" + node : "null";
+    if (typeof node !== "object") {
+        if (typeof node === "string" && (node.startsWith("encrypted_U") || node.startsWith("binary_U"))) {
+            return `"${node}"`;
+        }
+        return JSON.stringify(node);
+    }
+
+    let i, out;
+    if (Array.isArray(node)) {
+        out = "[";
+        for (i = 0; i < node.length; i++) {
+            if (i) out += ",";
+            out += stableStringify(node[i]) || "null";
+        }
+        return out + "]";
+    }
+
+    if (node === null) return "null";
+
+    if (seen.indexOf(node) !== -1) {
+        if (cycles) return JSON.stringify("__cycle__");
+        throw new TypeError("Converting circular structure to JSON");
+    }
+
+    const seenIndex = seen.push(node) - 1;
+    const keys = Object.keys(node).sort();
+    out = "";
+    for (i = 0; i < keys.length; i++) {
+        const key = keys[i]!;
+        const value = stableStringify(node[key]);
+
+        if (!value) continue;
+        if (out) out += ",";
+        out += JSON.stringify(key) + ":" + value;
+    }
+    seen.splice(seenIndex, 1);
+    return "{" + out + "}";
+}

package/src/group.test.ts

@@ -1,6 +1,10 @@
-import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream } from "./index";
+import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream, cojsonReady } from "./index";
 import { randomAnonymousAccountAndSessionID } from "./testUtils";
 
+beforeEach(async () => {
+    await cojsonReady;
+});
+
 test("Can create a CoMap in a group", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 

package/src/index.ts

@@ -18,11 +18,13 @@ import {
     agentSecretFromSecretSeed,
     secretSeedLength,
     shortHashLength,
+    cryptoReady
 } from "./crypto.js";
 import { connectedPeers } from "./streamUtils.js";
 import { AnonymousControlledAccount, ControlledAccount } from "./account.js";
 import { rawCoIDtoBytes, rawCoIDfromBytes } from "./ids.js";
 import { Group, expectGroupContent } from "./group.js";
+import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 
 import type { SessionID, AgentID } from "./ids.js";
 import type { CoID, CoValueImpl } from "./coValue.js";
@@ -50,6 +52,8 @@ export const cojsonInternals = {
     secretSeedLength,
     shortHashLength,
     expectGroupContent,
+    base64URLtoBytes,
+    bytesToBase64url
 };
 
 export {
@@ -66,6 +70,7 @@ export {
     CoValueCore,
     AnonymousControlledAccount,
     ControlledAccount,
+    cryptoReady as cojsonReady,
 };
 
 export type {

package/src/permissions.test.ts

@@ -17,7 +17,11 @@ import {
     groupWithTwoAdmins,
     groupWithTwoAdminsHighLevel,
 } from "./testUtils.js";
-import { AnonymousControlledAccount } from "./index.js";
+import { AnonymousControlledAccount, cojsonReady } from "./index.js";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Initial admin can add another admin to a group", () => {
     groupWithTwoAdmins();

package/src/sync.test.ts

@@ -1,14 +1,9 @@
 import { newRandomSessionID } from "./coValueCore.js";
 import { LocalNode } from "./node.js";
-import { Peer, PeerID, SyncMessage } from "./sync.js";
+import { SyncMessage } from "./sync.js";
 import { expectMap } from "./coValue.js";
 import { MapOpPayload } from "./coValues/coMap.js";
 import { Group } from "./group.js";
-import {
-    ReadableStream,
-    WritableStream,
-    TransformStream,
-} from "isomorphic-streams";
 import {
     randomAnonymousAccountAndSessionID,
     shouldNotResolve,
@@ -18,6 +13,11 @@ import {
     newStreamPair
 } from "./streamUtils.js";
 import { AccountID } from "./account.js";
+import { cojsonReady } from "./index.js";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Node replies with initial tx and header to empty subscribe", async () => {
     const [admin, session] = randomAnonymousAccountAndSessionID();