cojson 0.1.11 → 0.2.0

package/src/coValues/coStream.ts

@@ -1,9 +1,12 @@
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { CoID, ReadableCoValue, WriteableCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { Group } from "../group.js";
 import { SessionID } from "../ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
+import { AccountID } from "../index.js";
+import { isAccountID } from "../account.js";
+import { parseJSON } from "../jsonStringify.js";
 
 export type BinaryChunkInfo = {
     mimeType: string;
@@ -17,7 +20,7 @@ export type BinaryStreamStart = {
 
 export type BinaryStreamChunk = {
     type: "chunk";
-    chunk: `U${string}`;
+    chunk: `binary_U${string}`;
 };
 
 export type BinaryStreamEnd = {
@@ -40,7 +43,7 @@ export class CoStream<
     type = "costream" as const;
     core: CoValueCore;
     items: {
-        [key: SessionID]: T[];
+        [key: SessionID]: {item: T, madeAt: number}[];
     };
 
     constructor(core: CoValueCore) {
@@ -64,16 +67,17 @@ export class CoStream<
 
         for (const {
             txID,
+            madeAt,
             changes,
         } of this.core.getValidSortedTransactions()) {
-            for (const changeUntyped of changes) {
+            for (const changeUntyped of parseJSON(changes)) {
                 const change = changeUntyped as T;
                 let entries = this.items[txID.sessionID];
                 if (!entries) {
                     entries = [];
                     this.items[txID.sessionID] = entries;
                 }
-                entries.push(change);
+                entries.push({item: change, madeAt});
             }
         }
     }
@@ -87,13 +91,57 @@ export class CoStream<
             );
         }
 
-        return Object.values(this.items)[0];
+        return Object.values(this.items)[0]?.map(item => item.item);
+    }
+
+    getLastItemsPerAccount(): {[account: AccountID]: T | undefined} {
+        const result: {[account: AccountID]: {item: T, madeAt: number} | undefined} = {};
+
+        for (const [sessionID, items] of Object.entries(this.items)) {
+            const account = accountOrAgentIDfromSessionID(sessionID as SessionID);
+            if (!isAccountID(account)) continue;
+            if (items.length > 0) {
+                const lastItemOfSession = items[items.length - 1]!;
+                if (!result[account] || lastItemOfSession.madeAt > result[account]!.madeAt) {
+                    result[account] = lastItemOfSession;
+                }
+            }
+        }
+
+        return Object.fromEntries(Object.entries(result).map(([account, item]) =>
+            [account, item?.item]
+        ));
+    }
+
+    getLastItemFrom(account: AccountID): T | undefined {
+        let lastItem: {item: T, madeAt: number} | undefined;
+
+        for (const [sessionID, items] of Object.entries(this.items)) {
+            if (sessionID.startsWith(account)) {
+                if (items.length > 0) {
+                    const lastItemOfSession = items[items.length - 1]!;
+                    if (!lastItem || lastItemOfSession.madeAt > lastItem.madeAt) {
+                        lastItem = lastItemOfSession;
+                    }
+                }
+            }
+        }
+
+        return lastItem?.item;
+    }
+
+    getLastItemFromMe(): T | undefined {
+        const myAccountID = this.core.node.account.id;
+        if (!isAccountID(myAccountID)) return undefined;
+        return this.getLastItemFrom(myAccountID);
     }
 
     toJSON(): {
         [key: SessionID]: T[];
     } {
-        return this.items;
+        return Object.fromEntries(Object.entries(this.items).map(([sessionID, items]) =>
+            [sessionID, items.map(item => item.item)]
+        ));
     }
 
     subscribe(listener: (coMap: CoStream<T, Meta>) => void): () => void {
@@ -111,6 +159,8 @@ export class CoStream<
     }
 }
 
+const binary_U_prefixLength = 8; // "binary_U".length;
+
 export class BinaryCoStream<
         Meta extends BinaryCoStreamMeta = { type: "binary" }
     >
@@ -122,6 +172,7 @@ export class BinaryCoStream<
     getBinaryChunks():
         | (BinaryChunkInfo & { chunks: Uint8Array[]; finished: boolean })
         | undefined {
+        const before = performance.now();
         const items = this.getSingleStream();
 
         if (!items) return;
@@ -135,15 +186,13 @@ export class BinaryCoStream<
 
         const chunks: Uint8Array[] = [];
 
+        let finished = false;
+        let totalLength = 0;
+
         for (const item of items.slice(1)) {
             if (item.type === "end") {
-                return {
-                    mimeType: start.mimeType,
-                    fileName: start.fileName,
-                    totalSizeBytes: start.totalSizeBytes,
-                    chunks,
-                    finished: true,
-                };
+                finished = true;
+                break;
             }
 
             if (item.type !== "chunk") {
@@ -151,15 +200,25 @@ export class BinaryCoStream<
                 return undefined;
             }
 
-            chunks.push(base64URLtoBytes(item.chunk.slice(1)));
+            const chunk = base64URLtoBytes(
+                item.chunk.slice(binary_U_prefixLength)
+            );
+            totalLength += chunk.length;
+            chunks.push(chunk);
         }
 
+        const after = performance.now();
+        console.log(
+            "getBinaryChunks bandwidth in MB/s",
+            (1000 * totalLength) / (after - before) / (1024 * 1024)
+        );
+
         return {
             mimeType: start.mimeType,
             fileName: start.fileName,
             totalSizeBytes: start.totalSizeBytes,
             chunks,
-            finished: false,
+            finished,
         };
     }
 
@@ -206,10 +265,7 @@ export class WriteableBinaryCoStream<
     }
 
     /** @internal */
-    push(
-        item: BinaryStreamItem,
-        privacy: "private" | "trusting" = "private"
-    ) {
+    push(item: BinaryStreamItem, privacy: "private" | "trusting" = "private") {
         WriteableCoStream.prototype.push.call(this, item, privacy);
     }
 
@@ -230,13 +286,19 @@ export class WriteableBinaryCoStream<
         chunk: Uint8Array,
         privacy: "private" | "trusting" = "private"
     ) {
+        const before = performance.now();
         this.push(
             {
                 type: "chunk",
-                chunk: `U${bytesToBase64url(chunk)}`,
+                chunk: `binary_U${bytesToBase64url(chunk)}`,
             } satisfies BinaryStreamChunk,
             privacy
         );
+        const after = performance.now();
+        console.log(
+            "pushBinaryStreamChunk bandwidth in MB/s",
+            (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        );
     }
 
     endBinaryStream(privacy: "private" | "trusting" = "private") {

package/src/crypto.test.ts

@@ -21,6 +21,11 @@ import { xsalsa20_poly1305 } from "@noble/ciphers/salsa";
 import { blake3 } from "@noble/hashes/blake3";
 import stableStringify from "fast-json-stable-stringify";
 import { SessionID } from './ids.js';
+import { cojsonReady } from './index.js';
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Signatures round-trip and use stable stringify", () => {
     const data = { b: "world", a: "hello" };

package/src/crypto.ts

@@ -2,12 +2,39 @@ import { ed25519, x25519 } from "@noble/curves/ed25519";
 import { xsalsa20_poly1305, xsalsa20 } from "@noble/ciphers/salsa";
 import { JsonValue } from "./jsonValue.js";
 import { base58 } from "@scure/base";
-import stableStringify from "fast-json-stable-stringify";
-import { blake3 } from "@noble/hashes/blake3";
 import { randomBytes } from "@noble/ciphers/webcrypto/utils";
 import { AgentID, RawCoID, TransactionID } from "./ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 
+import { createBLAKE3 } from 'hash-wasm';
+import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
+
+let blake3Instance: Awaited<ReturnType<typeof createBLAKE3>>;
+let blake3HashOnce: (data: Uint8Array) => Uint8Array;
+let blake3HashOnceWithContext: (data: Uint8Array, {context}: {context: Uint8Array}) => Uint8Array;
+let blake3incrementalUpdateSLOW_WITH_DEVTOOLS: (state: Uint8Array, data: Uint8Array) => Uint8Array;
+let blake3digestForState: (state: Uint8Array) => Uint8Array;
+
+export const cryptoReady = new Promise<void>((resolve) => {
+    createBLAKE3().then(bl3 => {
+        blake3Instance = bl3;
+        blake3HashOnce = (data) => {
+            return bl3.init().update(data).digest('binary');
+        }
+        blake3HashOnceWithContext = (data, {context}) => {
+            return bl3.init().update(context).update(data).digest('binary');
+        }
+        blake3incrementalUpdateSLOW_WITH_DEVTOOLS = (state, data) => {
+            bl3.load(state).update(data);
+            return bl3.save();
+        }
+        blake3digestForState = (state) => {
+            return bl3.load(state).digest('binary');
+        }
+        resolve();
+    })
+});
+
 export type SignerSecret = `signerSecret_z${string}`;
 export type SignerID = `signer_z${string}`;
 export type Signature = `signature_z${string}`;
@@ -128,7 +155,7 @@ export function seal<T extends JsonValue>(
     to: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): Sealed<T> {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -153,7 +180,7 @@ export function unseal<T extends JsonValue>(
     from: SealerID,
     nOnceMaterial: { in: RawCoID; tx: TransactionID }
 ): T | undefined {
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -181,28 +208,32 @@ export type Hash = `hash_z${string}`;
 
 export function secureHash(value: JsonValue): Hash {
     return `hash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value)))
+        blake3HashOnce(textEncoder.encode(stableStringify(value)))
     )}`;
 }
 
 export class StreamingHash {
-    state: ReturnType<typeof blake3.create>;
+    state: Uint8Array;
 
-    constructor(fromClone?: ReturnType<typeof blake3.create>) {
-        this.state = fromClone || blake3.create({});
+    constructor(fromClone?: Uint8Array) {
+        this.state = fromClone || blake3Instance.init().save();
     }
 
     update(value: JsonValue) {
-        this.state.update(textEncoder.encode(stableStringify(value)));
+        const encoded = textEncoder.encode(stableStringify(value))
+        // const before = performance.now();
+        this.state = blake3incrementalUpdateSLOW_WITH_DEVTOOLS(this.state, encoded);
+        // const after = performance.now();
+        // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
     }
 
     digest(): Hash {
-        const hash = this.state.digest();
+        const hash = blake3digestForState(this.state);
         return `hash_z${base58.encode(hash)}`;
     }
 
     clone(): StreamingHash {
-        return new StreamingHash(this.state.clone());
+        return new StreamingHash(new Uint8Array(this.state));
     }
 }
 
@@ -211,7 +242,7 @@ export const shortHashLength = 19;
 
 export function shortHash(value: JsonValue): ShortHash {
     return `shortHash_z${base58.encode(
-        blake3(textEncoder.encode(stableStringify(value))).slice(
+        blake3HashOnce(textEncoder.encode(stableStringify(value))).slice(
             0,
             shortHashLength
         )
@@ -241,7 +272,7 @@ function encrypt<T extends JsonValue, N extends JsonValue>(
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -285,15 +316,15 @@ export function encryptKeySecret(keys: {
     };
 }
 
-function decrypt<T extends JsonValue, N extends JsonValue>(
+function decryptRaw<T extends JsonValue, N extends JsonValue>(
     encrypted: Encrypted<T, N>,
     keySecret: KeySecret,
     nOnceMaterial: N
-): T | undefined {
+): Stringified<T> {
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
-    const nOnce = blake3(
+    const nOnce = blake3HashOnce(
         textEncoder.encode(stableStringify(nOnceMaterial))
     ).slice(0, 24);
 
@@ -302,13 +333,31 @@ function decrypt<T extends JsonValue, N extends JsonValue>(
     );
     const plaintext = xsalsa20(keySecretBytes, nOnce, ciphertext);
 
+        return textDecoder.decode(plaintext) as Stringified<T>;
+
+}
+
+function decrypt<T extends JsonValue, N extends JsonValue>(
+    encrypted: Encrypted<T, N>,
+    keySecret: KeySecret,
+    nOnceMaterial: N
+): T | undefined {
     try {
-        return JSON.parse(textDecoder.decode(plaintext));
+        return parseJSON(decryptRaw(encrypted, keySecret, nOnceMaterial));
     } catch (e) {
+        console.error("Decryption error", e)
         return undefined;
     }
 }
 
+export function decryptRawForTransaction<T extends JsonValue>(
+    encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
+    keySecret: KeySecret,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID }
+): Stringified<T> | undefined {
+    return decryptRaw(encrypted, keySecret, nOnceMaterial);
+}
+
 export function decryptForTransaction<T extends JsonValue>(
     encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
     keySecret: KeySecret,
@@ -365,11 +414,11 @@ export function agentSecretFromSecretSeed(secretSeed: Uint8Array): AgentSecret {
     }
 
     return `sealerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("seal"),
         })
     )}/signerSecret_z${base58.encode(
-        blake3(secretSeed, {
+        blake3HashOnceWithContext(secretSeed, {
             context: textEncoder.encode("sign"),
         })
     )}`;

package/src/group.test.ts

@@ -1,6 +1,10 @@
-import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream } from "./index";
+import { LocalNode, CoMap, CoList, CoStream, BinaryCoStream, cojsonReady } from "./index";
 import { randomAnonymousAccountAndSessionID } from "./testUtils";
 
+beforeEach(async () => {
+    await cojsonReady;
+});
+
 test("Can create a CoMap in a group", () => {
     const node = new LocalNode(...randomAnonymousAccountAndSessionID());
 

package/src/group.ts

@@ -238,7 +238,7 @@ export class Group {
 
     /** Creates a new `CoMap` within this group, with the specified specialized
      *  `CoMap` type `M` and optional static metadata. */
-    createMap<M extends CoMap<{ [key: string]: JsonValue }, JsonObject | null>>(
+    createMap<M extends CoMap<{ [key: string]: JsonValue | undefined; }, JsonObject | null>>(
         meta?: M["meta"]
     ): M {
         return this.node

package/src/index.ts

@@ -18,12 +18,14 @@ import {
     agentSecretFromSecretSeed,
     secretSeedLength,
     shortHashLength,
+    cryptoReady
 } from "./crypto.js";
 import { connectedPeers } from "./streamUtils.js";
 import { AnonymousControlledAccount, ControlledAccount } from "./account.js";
 import { rawCoIDtoBytes, rawCoIDfromBytes } from "./ids.js";
 import { Group, expectGroupContent } from "./group.js";
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
+import { parseJSON } from "./jsonStringify.js";
 
 import type { SessionID, AgentID } from "./ids.js";
 import type { CoID, CoValueImpl } from "./coValue.js";
@@ -33,6 +35,7 @@ import type { SyncMessage, Peer } from "./sync.js";
 import type { AgentSecret } from "./crypto.js";
 import type { AccountID, Profile } from "./account.js";
 import type { InviteSecret } from "./group.js";
+import type * as Media from "./media.js";
 
 type Value = JsonValue | CoValueImpl;
 
@@ -52,7 +55,8 @@ export const cojsonInternals = {
     shortHashLength,
     expectGroupContent,
     base64URLtoBytes,
-    bytesToBase64url
+    bytesToBase64url,
+    parseJSON
 };
 
 export {
@@ -69,6 +73,7 @@ export {
     CoValueCore,
     AnonymousControlledAccount,
     ControlledAccount,
+    cryptoReady as cojsonReady,
 };
 
 export type {
@@ -88,6 +93,7 @@ export type {
     AgentSecret,
     InviteSecret,
     SyncMessage,
+    Media
 };
 
 // eslint-disable-next-line @typescript-eslint/no-namespace

package/src/jsonStringify.ts

@@ -0,0 +1,66 @@
+// adapted from fast-json-stable-stringify (https://github.com/epoberezkin/fast-json-stable-stringify)
+
+export type Stringified<T> = string & { __type: T };
+
+export function stableStringify<T>(data: T): Stringified<T>
+export function stableStringify(data: undefined): undefined
+export function stableStringify<T>(data: T | undefined): Stringified<T> | undefined {
+    const cycles = false;
+
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const seen: any[] = [];
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let node = data as any;
+
+    if (node && node.toJSON && typeof node.toJSON === "function") {
+        node = node.toJSON();
+    }
+
+    if (node === undefined) return;
+    if (typeof node == "number")
+        return (isFinite(node) ? "" + node : "null") as Stringified<T>;
+    if (typeof node !== "object") {
+        if (
+            typeof node === "string" &&
+            (node.startsWith("encrypted_U") || node.startsWith("binary_U"))
+        ) {
+            return `"${node}"` as Stringified<T>;
+        }
+        return JSON.stringify(node) as Stringified<T>;
+    }
+
+    let i, out;
+    if (Array.isArray(node)) {
+        out = "[";
+        for (i = 0; i < node.length; i++) {
+            if (i) out += ",";
+            out += stableStringify(node[i]) || "null";
+        }
+        return (out + "]") as Stringified<T>;
+    }
+
+    if (node === null) return "null" as Stringified<T>;
+
+    if (seen.indexOf(node) !== -1) {
+        if (cycles) return JSON.stringify("__cycle__") as Stringified<T>;
+        throw new TypeError("Converting circular structure to JSON");
+    }
+
+    const seenIndex = seen.push(node) - 1;
+    const keys = Object.keys(node).sort();
+    out = "";
+    for (i = 0; i < keys.length; i++) {
+        const key = keys[i]!;
+        const value = stableStringify(node[key]);
+
+        if (!value) continue;
+        if (out) out += ",";
+        out += JSON.stringify(key) + ":" + value;
+    }
+    seen.splice(seenIndex, 1);
+    return ("{" + out + "}") as Stringified<T>;
+}
+
+export function parseJSON<T>(json: Stringified<T>): T {
+    return JSON.parse(json);
+}

package/src/jsonValue.ts

@@ -3,4 +3,4 @@ import { RawCoID } from './ids.js';
 export type JsonAtom = string | number | boolean | null;
 export type JsonValue = JsonAtom | JsonArray | JsonObject | RawCoID;
 export type JsonArray = JsonValue[];
-export type JsonObject = { [key: string]: JsonValue; };
+export type JsonObject = { [key: string]: JsonValue | undefined; };

package/src/media.ts

@@ -0,0 +1,9 @@
+import { CoMap } from './coValues/coMap.js'
+import { CoID } from './coValue.js'
+import { BinaryCoStream } from './coValues/coStream.js'
+
+export type ImageDefinition = CoMap<{
+    originalSize: [number, number];
+    placeholderDataURL?: string;
+    [res: `${number}x${number}`]: CoID<BinaryCoStream>;
+}>;

package/src/node.ts

@@ -208,7 +208,7 @@ export class LocalNode {
                     reject(
                         new Error("Couldn't find invite before timeout")
                     ),
-                1000
+                2000
             );
         });
 

package/src/permissions.test.ts

@@ -17,7 +17,11 @@ import {
     groupWithTwoAdmins,
     groupWithTwoAdminsHighLevel,
 } from "./testUtils.js";
-import { AnonymousControlledAccount } from "./index.js";
+import { AnonymousControlledAccount, cojsonReady } from "./index.js";
+
+beforeEach(async () => {
+    await cojsonReady;
+});
 
 test("Initial admin can add another admin to a group", () => {
     groupWithTwoAdmins();

package/src/permissions.ts

@@ -15,6 +15,7 @@ import {
     AccountID,
     Profile,
 } from "./account.js";
+import { parseJSON } from "./jsonStringify.js";
 
 export type PermissionsDef =
     | { type: "group"; initialAdmin: AccountID | AgentID }
@@ -76,11 +77,13 @@ export function determineValidTransactions(
             // console.log("before", { memberState, validTransactions });
             const transactor = accountOrAgentIDfromSessionID(sessionID);
 
-            const change = tx.changes[0] as
+            const changes = parseJSON(tx.changes)
+
+            const change = changes[0] as
                 | MapOpPayload<AccountID | AgentID, Role>
                 | MapOpPayload<"readKey", JsonValue>
                 | MapOpPayload<"profile", CoID<Profile>>;
-            if (tx.changes.length !== 1) {
+            if (changes.length !== 1) {
                 console.warn("Group transaction must have exactly one change");
                 continue;
             }