cojson 0.1.12 → 0.2.1

package/src/crypto.ts

@@ -7,7 +7,7 @@ import { AgentID, RawCoID, TransactionID } from "./ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
 
 import { createBLAKE3 } from 'hash-wasm';
-import { stableStringify } from "./fastJsonStableStringify.js";
+import { Stringified, parseJSON, stableStringify } from "./jsonStringify.js";
 
 let blake3Instance: Awaited<ReturnType<typeof createBLAKE3>>;
 let blake3HashOnce: (data: Uint8Array) => Uint8Array;
@@ -316,11 +316,11 @@ export function encryptKeySecret(keys: {
     };
 }
 
-function decrypt<T extends JsonValue, N extends JsonValue>(
+function decryptRaw<T extends JsonValue, N extends JsonValue>(
     encrypted: Encrypted<T, N>,
     keySecret: KeySecret,
     nOnceMaterial: N
-): T | undefined {
+): Stringified<T> {
     const keySecretBytes = base58.decode(
         keySecret.substring("keySecret_z".length)
     );
@@ -333,13 +333,31 @@ function decrypt<T extends JsonValue, N extends JsonValue>(
     );
     const plaintext = xsalsa20(keySecretBytes, nOnce, ciphertext);
 
+        return textDecoder.decode(plaintext) as Stringified<T>;
+
+}
+
+function decrypt<T extends JsonValue, N extends JsonValue>(
+    encrypted: Encrypted<T, N>,
+    keySecret: KeySecret,
+    nOnceMaterial: N
+): T | undefined {
     try {
-        return JSON.parse(textDecoder.decode(plaintext));
+        return parseJSON(decryptRaw(encrypted, keySecret, nOnceMaterial));
     } catch (e) {
+        console.error("Decryption error", e)
         return undefined;
     }
 }
 
+export function decryptRawForTransaction<T extends JsonValue>(
+    encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
+    keySecret: KeySecret,
+    nOnceMaterial: { in: RawCoID; tx: TransactionID }
+): Stringified<T> | undefined {
+    return decryptRaw(encrypted, keySecret, nOnceMaterial);
+}
+
 export function decryptForTransaction<T extends JsonValue>(
     encrypted: Encrypted<T, { in: RawCoID; tx: TransactionID }>,
     keySecret: KeySecret,

package/src/group.ts

@@ -238,7 +238,7 @@ export class Group {
 
     /** Creates a new `CoMap` within this group, with the specified specialized
      *  `CoMap` type `M` and optional static metadata. */
-    createMap<M extends CoMap<{ [key: string]: JsonValue }, JsonObject | null>>(
+    createMap<M extends CoMap<{ [key: string]: JsonValue | undefined; }, JsonObject | null>>(
         meta?: M["meta"]
     ): M {
         return this.node

package/src/index.ts

@@ -1,4 +1,4 @@
-import { CoValueCore, newRandomSessionID } from "./coValueCore.js";
+import { CoValueCore, newRandomSessionID, MAX_RECOMMENDED_TX_SIZE } from "./coValueCore.js";
 import { LocalNode } from "./node.js";
 import type { CoValue, ReadableCoValue } from "./coValue.js";
 import { CoMap, WriteableCoMap } from "./coValues/coMap.js";
@@ -25,6 +25,7 @@ import { AnonymousControlledAccount, ControlledAccount } from "./account.js";
 import { rawCoIDtoBytes, rawCoIDfromBytes } from "./ids.js";
 import { Group, expectGroupContent } from "./group.js";
 import { base64URLtoBytes, bytesToBase64url } from "./base64url.js";
+import { parseJSON } from "./jsonStringify.js";
 
 import type { SessionID, AgentID } from "./ids.js";
 import type { CoID, CoValueImpl } from "./coValue.js";
@@ -34,6 +35,7 @@ import type { SyncMessage, Peer } from "./sync.js";
 import type { AgentSecret } from "./crypto.js";
 import type { AccountID, Profile } from "./account.js";
 import type { InviteSecret } from "./group.js";
+import type * as Media from "./media.js";
 
 type Value = JsonValue | CoValueImpl;
 
@@ -53,7 +55,8 @@ export const cojsonInternals = {
     shortHashLength,
     expectGroupContent,
     base64URLtoBytes,
-    bytesToBase64url
+    bytesToBase64url,
+    parseJSON
 };
 
 export {
@@ -71,6 +74,7 @@ export {
     AnonymousControlledAccount,
     ControlledAccount,
     cryptoReady as cojsonReady,
+    MAX_RECOMMENDED_TX_SIZE
 };
 
 export type {
@@ -90,6 +94,7 @@ export type {
     AgentSecret,
     InviteSecret,
     SyncMessage,
+    Media
 };
 
 // eslint-disable-next-line @typescript-eslint/no-namespace

package/src/{fastJsonStableStringify.ts → jsonStringify.ts}

@@ -1,24 +1,32 @@
 // adapted from fast-json-stable-stringify (https://github.com/epoberezkin/fast-json-stable-stringify)
 
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-export function stableStringify(data: any): string | undefined {
+export type Stringified<T> = string & { __type: T };
+
+export function stableStringify<T>(data: T): Stringified<T>
+export function stableStringify(data: undefined): undefined
+export function stableStringify<T>(data: T | undefined): Stringified<T> | undefined {
     const cycles = false;
 
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const seen: any[] = [];
-    let node = data;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    let node = data as any;
 
     if (node && node.toJSON && typeof node.toJSON === "function") {
         node = node.toJSON();
     }
 
     if (node === undefined) return;
-    if (typeof node == "number") return isFinite(node) ? "" + node : "null";
+    if (typeof node == "number")
+        return (isFinite(node) ? "" + node : "null") as Stringified<T>;
     if (typeof node !== "object") {
-        if (typeof node === "string" && (node.startsWith("encrypted_U") || node.startsWith("binary_U"))) {
-            return `"${node}"`;
+        if (
+            typeof node === "string" &&
+            (node.startsWith("encrypted_U") || node.startsWith("binary_U"))
+        ) {
+            return `"${node}"` as Stringified<T>;
         }
-        return JSON.stringify(node);
+        return JSON.stringify(node) as Stringified<T>;
     }
 
     let i, out;
@@ -28,13 +36,13 @@ export function stableStringify(data: any): string | undefined {
             if (i) out += ",";
             out += stableStringify(node[i]) || "null";
         }
-        return out + "]";
+        return (out + "]") as Stringified<T>;
     }
 
-    if (node === null) return "null";
+    if (node === null) return "null" as Stringified<T>;
 
     if (seen.indexOf(node) !== -1) {
-        if (cycles) return JSON.stringify("__cycle__");
+        if (cycles) return JSON.stringify("__cycle__") as Stringified<T>;
         throw new TypeError("Converting circular structure to JSON");
     }
 
@@ -50,5 +58,9 @@ export function stableStringify(data: any): string | undefined {
         out += JSON.stringify(key) + ":" + value;
     }
     seen.splice(seenIndex, 1);
-    return "{" + out + "}";
+    return ("{" + out + "}") as Stringified<T>;
 }
+
+export function parseJSON<T>(json: Stringified<T>): T {
+    return JSON.parse(json);
+}

package/src/jsonValue.ts

@@ -3,4 +3,4 @@ import { RawCoID } from './ids.js';
 export type JsonAtom = string | number | boolean | null;
 export type JsonValue = JsonAtom | JsonArray | JsonObject | RawCoID;
 export type JsonArray = JsonValue[];
-export type JsonObject = { [key: string]: JsonValue; };
+export type JsonObject = { [key: string]: JsonValue | undefined; };

package/src/media.ts

@@ -0,0 +1,9 @@
+import { CoMap } from './coValues/coMap.js'
+import { CoID } from './coValue.js'
+import { BinaryCoStream } from './coValues/coStream.js'
+
+export type ImageDefinition = CoMap<{
+    originalSize: [number, number];
+    placeholderDataURL?: string;
+    [res: `${number}x${number}`]: CoID<BinaryCoStream>;
+}>;

package/src/node.ts

@@ -208,7 +208,7 @@ export class LocalNode {
                     reject(
                         new Error("Couldn't find invite before timeout")
                     ),
-                1000
+                2000
             );
         });
 

package/src/permissions.ts

@@ -15,6 +15,7 @@ import {
     AccountID,
     Profile,
 } from "./account.js";
+import { parseJSON } from "./jsonStringify.js";
 
 export type PermissionsDef =
     | { type: "group"; initialAdmin: AccountID | AgentID }
@@ -76,11 +77,13 @@ export function determineValidTransactions(
             // console.log("before", { memberState, validTransactions });
             const transactor = accountOrAgentIDfromSessionID(sessionID);
 
-            const change = tx.changes[0] as
+            const changes = parseJSON(tx.changes)
+
+            const change = changes[0] as
                 | MapOpPayload<AccountID | AgentID, Role>
                 | MapOpPayload<"readKey", JsonValue>
                 | MapOpPayload<"profile", CoID<Profile>>;
-            if (tx.changes.length !== 1) {
+            if (changes.length !== 1) {
                 console.warn("Group transaction must have exactly one change");
                 continue;
             }

package/src/streamUtils.ts

@@ -34,7 +34,14 @@ export function connectedPeers(
                     trace &&
                         console.debug(
                             `${peer2id} -> ${peer1id}`,
-                            JSON.stringify(chunk, null, 2)
+                            JSON.stringify(
+                                chunk,
+                                (k, v) =>
+                                    (k === "changes" || k === "encryptedChanges")
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                                2
+                            )
                         );
                     controller.enqueue(chunk);
                 },
@@ -52,7 +59,14 @@ export function connectedPeers(
                     trace &&
                         console.debug(
                             `${peer1id} -> ${peer2id}`,
-                            JSON.stringify(chunk, null, 2)
+                            JSON.stringify(
+                                chunk,
+                                (k, v) =>
+                                    (k === "changes" || k === "encryptedChanges")
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                                2
+                            )
                         );
                     controller.enqueue(chunk);
                 },
@@ -102,16 +116,22 @@ export function newStreamPair<T>(): [ReadableStream<T>, WritableStream<T>] {
         },
     });
 
+    let lastWritePromise = Promise.resolve();
+
     const writable = new WritableStream<T>({
         async write(chunk) {
             const enqueue = await enqueuePromise;
             if (readerClosed) {
                 throw new Error("Reader closed");
             } else {
-                // make sure write resolves before corresponding read
-                setTimeout(() => {
-                    enqueue(chunk);
-                })
+                // make sure write resolves before corresponding read, but make sure writes are still in order
+                await lastWritePromise;
+                lastWritePromise = new Promise((resolve) => {
+                    setTimeout(() => {
+                        enqueue(chunk);
+                        resolve();
+                    });
+                });
             }
         },
         async abort(reason) {

package/src/sync.test.ts

@@ -8,12 +8,10 @@ import {
     randomAnonymousAccountAndSessionID,
     shouldNotResolve,
 } from "./testUtils.js";
-import {
-    connectedPeers,
-    newStreamPair
-} from "./streamUtils.js";
+import { connectedPeers, newStreamPair } from "./streamUtils.js";
 import { AccountID } from "./account.js";
 import { cojsonReady } from "./index.js";
+import { stableStringify } from "./jsonStringify.js";
 
 beforeEach(async () => {
     await cojsonReady;
@@ -84,13 +82,13 @@ test("Node replies with initial tx and header to empty subscribe", async () => {
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[0]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "hello",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -162,13 +160,13 @@ test("Node replies with only new tx to subscribe with some known state", async (
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[1]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "goodbye",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -251,13 +249,13 @@ test("After subscribing, node sends own known state and new txs to peer", async
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[0]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "hello",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -283,13 +281,13 @@ test("After subscribing, node sends own known state and new txs to peer", async
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[1]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "goodbye",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -362,13 +360,13 @@ test("Client replies with known new content to tellKnownState from server", asyn
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[0]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "hello",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -438,8 +436,9 @@ test("No matter the optimistic known state, node respects invalid known state me
         editable.set("goodbye", "world", "trusting");
     });
 
-    const _mapEditMsg1 = await reader.read();
-    const _mapEditMsg2 = await reader.read();
+    const _mapEditMsgs = await reader.read();
+
+    console.log("Sending correction");
 
     await writer.write({
         action: "known",
@@ -465,13 +464,13 @@ test("No matter the optimistic known state, node respects invalid known state me
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[1]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "goodbye",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:
@@ -568,13 +567,13 @@ test("If we add a server peer, all updates to all coValues are sent to it, even
                         privacy: "trusting" as const,
                         madeAt: map.core.sessions[node.currentSessionID]!
                             .transactions[0]!.madeAt,
-                        changes: [
+                        changes: stableStringify([
                             {
                                 op: "set",
                                 key: "hello",
                                 value: "world",
                             } satisfies MapOpPayload<string, string>,
-                        ],
+                        ]),
                     },
                 ],
                 lastSignature:

package/src/sync.ts

@@ -9,7 +9,7 @@ import {
     WritableStreamDefaultWriter,
 } from "isomorphic-streams";
 import { RawCoID, SessionID } from "./ids.js";
-import { stableStringify } from "./fastJsonStableStringify.js";
+import { stableStringify } from "./jsonStringify.js";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -215,14 +215,32 @@ export class SyncManager {
             await this.sendNewContentIncludingDependencies(id, peer);
         }
 
-        const newContent = coValue.newContentSince(
+        const newContentPieces = coValue.newContentSince(
             peer.optimisticKnownStates[id]
         );
 
-        if (newContent) {
-            await this.trySendToPeer(peer, newContent);
+        if (newContentPieces) {
+            const optimisticKnownStateBefore =
+                peer.optimisticKnownStates[id] || emptyKnownState(id);
+
+            const sendPieces = async () => {
+                for (const [i, piece] of newContentPieces.entries()) {
+                    // console.log(
+                    //     `${id} -> ${peer.id}: Sending content piece ${i + 1}/${newContentPieces.length} header: ${!!piece.header}`,
+                    //     // Object.values(piece.new).map((s) => s.newTransactions)
+                    // );
+                    await this.trySendToPeer(peer, piece);
+                }
+            };
+
+            sendPieces().catch((e) => {
+                console.error("Error sending new content piece, retrying", e);
+                peer.optimisticKnownStates[id] = optimisticKnownStateBefore;
+                return this.sendNewContentIncludingDependencies(id, peer);
+            });
+
             peer.optimisticKnownStates[id] = combinedKnownStates(
-                peer.optimisticKnownStates[id] || emptyKnownState(id),
+                optimisticKnownStateBefore,
                 coValue.knownState()
             );
         }
@@ -261,6 +279,9 @@ export class SyncManager {
                 for await (const msg of peerState.incoming) {
                     try {
                         await this.handleSyncMessage(msg, peerState);
+                        await new Promise<void>((resolve) => {
+                            setTimeout(resolve, 0);
+                        });
                     } catch (e) {
                         console.error(
                             `Error reading from peer ${peer.id}, handling msg`,
@@ -269,7 +290,6 @@ export class SyncManager {
                         );
                     }
                 }
-                console.log("DONE!!!");
             } catch (e) {
                 console.error(`Error reading from peer ${peer.id}`, e);
             }
@@ -446,6 +466,10 @@ export class SyncManager {
             const newTransactions =
                 newContentForSession.newTransactions.slice(alreadyKnownOffset);
 
+            if (newTransactions.length === 0) {
+                continue;
+            }
+
             const before = performance.now();
             const success = await coValue.tryAddTransactionsAsync(
                 sessionID,
@@ -455,20 +479,26 @@ export class SyncManager {
             );
             const after = performance.now();
             if (after - before > 10) {
-                const totalTxLength = newTransactions.map(t => stableStringify(t)!.length).reduce((a, b) => a + b, 0);
+                const totalTxLength = newTransactions
+                    .map((t) =>
+                        t.privacy === "private"
+                            ? t.encryptedChanges.length
+                            : t.changes.length
+                    )
+                    .reduce((a, b) => a + b, 0);
                 console.log(
-                    "Adding incoming transactions took",
-                    after - before,
-                    "ms",
-                    totalTxLength,
-                    "bytes = ",
-                    "bandwidth: MB/s",
-                    (1000 * totalTxLength / (after - before)) / (1024 * 1024)
+                    `Adding incoming transactions took ${(
+                        after - before
+                    ).toFixed(2)}ms for ${totalTxLength} bytes = bandwidth: ${(
+                        (1000 * totalTxLength) /
+                        (after - before) /
+                        (1024 * 1024)
+                    ).toFixed(2)} MB/s`
                 );
             }
 
             if (!success) {
-                console.error("Failed to add transactions", newTransactions);
+                console.error("Failed to add transactions", msg.id, newTransactions);
                 continue;
             }
 
@@ -493,18 +523,9 @@ export class SyncManager {
     }
 
     async handleCorrection(msg: KnownStateMessage, peer: PeerState) {
-        const coValue = this.local.expectCoValueLoaded(msg.id);
+        peer.optimisticKnownStates[msg.id] = msg;
 
-        peer.optimisticKnownStates[msg.id] = combinedKnownStates(
-            msg,
-            coValue.knownState()
-        );
-
-        const newContent = coValue.newContentSince(msg);
-
-        if (newContent) {
-            await this.trySendToPeer(peer, newContent);
-        }
+        return this.sendNewContentIncludingDependencies(msg.id, peer);
     }
 
     handleUnsubscribe(_msg: DoneMessage) {

package/dist/fastJsonStableStringify.d.ts

@@ -1 +0,0 @@
-export declare function stableStringify(data: any): string | undefined;

package/dist/fastJsonStableStringify.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"fastJsonStableStringify.js","sourceRoot":"","sources":["../src/fastJsonStableStringify.ts"],"names":[],"mappings":"AAAA,sGAAsG;AAEtG,8DAA8D;AAC9D,MAAM,UAAU,eAAe,CAAC,IAAS;IACrC,MAAM,MAAM,GAAG,KAAK,CAAC;IAErB,8DAA8D;IAC9D,MAAM,IAAI,GAAU,EAAE,CAAC;IACvB,IAAI,IAAI,GAAG,IAAI,CAAC;IAEhB,IAAI,IAAI,IAAI,IAAI,CAAC,MAAM,IAAI,OAAO,IAAI,CAAC,MAAM,KAAK,UAAU,EAAE;QAC1D,IAAI,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;KACxB;IAED,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO;IAC/B,IAAI,OAAO,IAAI,IAAI,QAAQ;QAAE,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;QAC1B,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,EAAE;YAC7F,OAAO,IAAI,IAAI,GAAG,CAAC;SACtB;QACD,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;KAC/B;IAED,IAAI,CAAC,EAAE,GAAG,CAAC;IACX,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;QACrB,GAAG,GAAG,GAAG,CAAC;QACV,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC9B,IAAI,CAAC;gBAAE,GAAG,IAAI,GAAG,CAAC;YAClB,GAAG,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;SAC7C;QACD,OAAO,GAAG,GAAG,GAAG,CAAC;KACpB;IAED,IAAI,IAAI,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAEjC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE;QAC3B,IAAI,MAAM;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QAC/C,MAAM,IAAI,SAAS,CAAC,uCAAuC,CAAC,CAAC;KAChE;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IACtC,GAAG,GAAG,EAAE,CAAC;IACT,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACrB,MAAM,KAAK,GAAG,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QAEzC,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,IAAI,GAAG;YAAE,GAAG,IAAI,GAAG,CAAC;QACpB,GAAG,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,GAAG,GAAG,KAAK,CAAC;KAC5C;IACD,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IAC1B,OAAO,GAAG,GAAG,GAAG,GAAG,GAAG,CAAC;AAC3B,CAAC"}