cojson 0.1.11 → 0.1.12

package/src/crypto.ts

@@ -2,12 +2,39 @@ import { ed25519, x25519 } from "@noble/curves/ed25519";
 import { xsalsa20_poly1305, xsalsa20 } from "@noble/ciphers/salsa";
 import { JsonValue } from "./jsonValue.js";
 import { base58 } from "@scure/base";
-import stableStringify from "fast-json-stable-stringify";
-import { blake3 } from "@noble/hashes/blake3";
 import { randomBytes } from "@noble/ciphers/webcrypto/utils";
 import { AgentID, RawCoID, TransactionID } from "./ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 
+import { createBLAKE3 } from 'hash-wasm';
+import { stableStringify } from "./fastJsonStableStringify.js";
+
+let blake3Instance: Awaited<ReturnType<typeof createBLAKE3>>;
+let blake3HashOnce: (data: Uint8Array) => Uint8Array;
+let blake3HashOnceWithContext: (data: Uint8Array, {context}: {context: Uint8Array}) => Uint8Array;
+let blake3incrementalUpdateSLOW_WITH_DEVTOOLS: (state: Uint8Array, data: Uint8Array) => Uint8Array;
+let blake3digestForState: (state: Uint8Array) => Uint8Array;
+
+export const cryptoReady = new Promise<void>((resolve) => {
+    createBLAKE3().then(bl3 => {
+        blake3Instance = bl3;
+        blake3HashOnce = (data) => {
+            return bl3.init().update(data).digest('binary');
+        }
+        blake3HashOnceWithContext = (data, {context}) => {
+            return bl3.init().update(context).update(data).digest('binary');
+        }
+        blake3incrementalUpdateSLOW_WITH_DEVTOOLS = (state, data) => {
+            bl3.load(state).update(data);
+            return bl3.save();
+        }
+        blake3digestForState = (state) => {
+            return bl3.load(state).digest('binary');
+        }
+        resolve();
+    })
+});
+
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
 export type Signature = `signature_z${string}`;
@@ -128,7 +155,7 @@ export function seal<T extends JsonValue>(
     to: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): Sealed<T> {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -153,7 +180,7 @@ export function unseal<T extends JsonValue>(
     from: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): T | undefined {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -181,28 +208,32 @@ export type Hash = `hash_z${string}`;
 
 export function secureHash(value: JsonValue): Hash {
     return `hash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value)))
+        blake3HashOnce(textEncoder.encode(stableStringify(value)))
     )}`;
 }
 
 export class StreamingHash {
-    state: ReturnType<typeof blake3.create>;
+    state: Uint8Array;
 
-    constructor(fromClone?: ReturnType<typeof blake3.create>) {
-        this.state = fromClone || blake3.create({});
+    constructor(fromClone?: Uint8Array) {
+        this.state = fromClone || blake3Instance.init().save();
     }
 
     update(value: JsonValue) {
-        this.state.update(textEncoder.encode(stableStringify(value)));
+        const encoded = textEncoder.encode(stableStringify(value))
+        // const before = performance.now();
+        this.state = blake3incrementalUpdateSLOW_WITH_DEVTOOLS(this.state, encoded);
+        // const after = performance.now();
+        // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
     }
 
     digest(): Hash {
-        const hash = this.state.digest();
+        const hash = blake3digestForState(this.state);
         return `hash_z${base58.encode(hash)}`;
     }
 
     clone(): StreamingHash {
-        return new StreamingHash(this.state.clone());
+        return new StreamingHash(new Uint8Array(this.state));
     }
 }
 
@@ -211,7 +242,7 @@ export const shortHashLength = 19;
 
 export function shortHash(value: JsonValue): ShortHash {
     return `shortHash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value))).slice(
+        blake3HashOnce(textEncoder.encode(stableStringify(value))).slice(
             0,
             shortHashLength
         )
@@ -241,7 +272,7 @@ function encrypt<T extends JsonValue, N extends JsonValue>(
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -293,7 +324,7 @@ function decrypt<T extends JsonValue, N extends JsonValue>(
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -365,11 +396,11 @@ export function agentSecretFromSecretSeed(secretSeed: Uint8Array): AgentSecret {
     }
 
     return `sealerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("seal"),
         })
     )}/signerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("sign"),
         })
     )}`;

package/src/fastJsonStableStringify.ts

@@ -0,0 +1,54 @@
+// adapted from fast-json-stable-stringify (https://github.com/epoberezkin/fast-json-stable-stringify)
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+export function stableStringify(data: any): string | undefined {
+    const cycles = false;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const seen: any[] = [];
+    let node = data;
+
+    if (node && node.toJSON && typeof node.toJSON === "function") {
+        node = node.toJSON();
+    }
+
+    if (node === undefined) return;
+    if (typeof node == "number") return isFinite(node) ? "" + node : "null";
+    if (typeof node !== "object") {
+        if (typeof node === "string" && (node.startsWith("encrypted_U") || node.startsWith("binary_U"))) {
+            return `"${node}"`;
+        }
+        return JSON.stringify(node);
+    }
+
+    let i, out;
+    if (Array.isArray(node)) {
+        out = "[";
+        for (i = 0; i < node.length; i++) {
+            if (i) out += ",";
+            out += stableStringify(node[i]) || "null";
+        }
+        return out + "]";
+    }
+
+    if (node === null) return "null";
+
+    if (seen.indexOf(node) !== -1) {
+        if (cycles) return JSON.stringify("__cycle__");
+        throw new TypeError("Converting circular structure to JSON");
+    }
+
+    const seenIndex = seen.push(node) - 1;
+    const keys = Object.keys(node).sort();
+    out = "";
+    for (i = 0; i < keys.length; i++) {
+        const key = keys[i]!;
+        const value = stableStringify(node[key]);
+
+        if (!value) continue;
+        if (out) out += ",";
+        out += JSON.stringify(key) + ":" + value;
+    }
+    seen.splice(seenIndex, 1);
+    return "{" + out + "}";
+}

package/src/group.test.ts

@@ -1,6 +1,10 @@
-import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream } from "./index";
+import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream, cojsonReady } from "./index";
 import { randomAnonymousAccountAndSessionID } from "./testUtils";
 
+beforeEach(async () => {
+    await cojsonReady;
+});
+
 test("Can create a CoMap in a group", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 

package/src/index.ts

@@ -18,6 +18,7 @@ import {
     agentSecretFromSecretSeed,
     secretSeedLength,
     shortHashLength,
+    cryptoReady
 } from "./crypto.js";
 import { connectedPeers } from "./streamUtils.js";
 import { AnonymousControlledAccount, ControlledAccount } from "./account.js";
@@ -69,6 +70,7 @@ export {
     CoValueCore,
     AnonymousControlledAccount,
     ControlledAccount,
+    cryptoReady as cojsonReady,
 };
 
 export type {

package/src/permissions.test.ts

@@ -17,7 +17,11 @@ import {
     groupWithTwoAdmins,
     groupWithTwoAdminsHighLevel,
 } from "./testUtils.js";
-import { AnonymousControlledAccount } from "./index.js";
+import { AnonymousControlledAccount, cojsonReady } from "./index.js";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Initial admin can add another admin to a group", () => {
     groupWithTwoAdmins();

package/src/sync.test.ts

@@ -1,14 +1,9 @@
 import { newRandomSessionID } from "./coValueCore.js";
 import { LocalNode } from "./node.js";
-import { Peer, PeerID, SyncMessage } from "./sync.js";
+import { SyncMessage } from "./sync.js";
 import { expectMap } from "./coValue.js";
 import { MapOpPayload } from "./coValues/coMap.js";
 import { Group } from "./group.js";
-import {
-    ReadableStream,
-    WritableStream,
-    TransformStream,
-} from "isomorphic-streams";
 import {
     randomAnonymousAccountAndSessionID,
     shouldNotResolve,
@@ -18,6 +13,11 @@ import {
     newStreamPair
 } from "./streamUtils.js";
 import { AccountID } from "./account.js";
+import { cojsonReady } from "./index.js";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Node replies with initial tx and header to empty subscribe", async () => {
     const [admin, session] = randomAnonymousAccountAndSessionID();

package/src/sync.ts

@@ -9,6 +9,7 @@ import {
     WritableStreamDefaultWriter,
 } from "isomorphic-streams";
 import { RawCoID, SessionID } from "./ids.js";
+import { stableStringify } from "./fastJsonStableStringify.js";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -445,12 +446,26 @@ export class SyncManager {
             const newTransactions =
                 newContentForSession.newTransactions.slice(alreadyKnownOffset);
 
-            const success = coValue.tryAddTransactions(
+            const before = performance.now();
+            const success = await coValue.tryAddTransactionsAsync(
                 sessionID,
                 newTransactions,
                 undefined,
                 newContentForSession.lastSignature
             );
+            const after = performance.now();
+            if (after - before > 10) {
+                const totalTxLength = newTransactions.map(t => stableStringify(t)!.length).reduce((a, b) => a + b, 0);
+                console.log(
+                    "Adding incoming transactions took",
+                    after - before,
+                    "ms",
+                    totalTxLength,
+                    "bytes = ",
+                    "bandwidth: MB/s",
+                    (1000 * totalTxLength / (after - before)) / (1024 * 1024)
+                );
+            }
 
             if (!success) {
                 console.error("Failed to add transactions", newTransactions);