cojson 0.1.12 → 0.2.1

package/src/coValueCore.ts

@@ -14,11 +14,11 @@ import {
     sign,
     verify,
     encryptForTransaction,
-    decryptForTransaction,
     KeyID,
     decryptKeySecret,
     getAgentSignerID,
     getAgentSealerID,
+    decryptRawForTransaction,
 } from "./crypto.js";
 import { JsonObject, JsonValue } from "./jsonValue.js";
 import { base58 } from "@scure/base";
@@ -32,10 +32,10 @@ import { LocalNode } from "./node.js";
 import { CoValueKnownState, NewContentMessage } from "./sync.js";
 import { AgentID, RawCoID, SessionID, TransactionID } from "./ids.js";
 import { CoList } from "./coValues/coList.js";
-import {
-    AccountID,
-    GeneralizedControlledAccount,
-} from "./account.js";
+import { AccountID, GeneralizedControlledAccount } from "./account.js";
+import { Stringified, stableStringify } from "./jsonStringify.js";
+
+export const MAX_RECOMMENDED_TX_SIZE = 100 * 1024;
 
 export type CoValueHeader = {
     type: CoValueImpl["type"];
@@ -64,6 +64,7 @@ type SessionLog = {
     transactions: Transaction[];
     lastHash?: Hash;
     streamingHash: StreamingHash;
+    signatureAfter: { [txIdx: number]: Signature | undefined };
     lastSignature: Signature;
 };
 
@@ -80,14 +81,14 @@ export type PrivateTransaction = {
 export type TrustingTransaction = {
     privacy: "trusting";
     madeAt: number;
-    changes: JsonValue[];
+    changes: Stringified<JsonValue[]>;
 };
 
 export type Transaction = PrivateTransaction | TrustingTransaction;
 
 export type DecryptedTransaction = {
     txID: TransactionID;
-    changes: JsonValue[];
+    changes: Stringified<JsonValue[]>;
     madeAt: number;
 };
 
@@ -100,7 +101,11 @@ export class CoValueCore {
     _sessions: { [key: SessionID]: SessionLog };
     _cachedContent?: CoValueImpl;
     listeners: Set<(content?: CoValueImpl) => void> = new Set();
-    _decryptionCache: {[key: Encrypted<JsonValue[], JsonValue>]: JsonValue[] | undefined} = {}
+    _decryptionCache: {
+        [key: Encrypted<JsonValue[], JsonValue>]:
+            | Stringified<JsonValue[]>
+            | undefined;
+    } = {};
 
     constructor(
         header: CoValueHeader,
@@ -209,7 +214,8 @@ export class CoValueCore {
         // const beforeVerify = performance.now();
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
-                "Invalid signature",
+                "Invalid signature in",
+                this.id,
                 newSignature,
                 expectedNewHash,
                 signerID
@@ -222,25 +228,13 @@ export class CoValueCore {
         //     afterVerify - beforeVerify
         // );
 
-        const transactions = this.sessions[sessionID]?.transactions ?? [];
-
-        transactions.push(...newTransactions);
-
-        this._sessions[sessionID] = {
-            transactions,
-            lastHash: expectedNewHash,
-            streamingHash: newStreamingHash,
-            lastSignature: newSignature,
-        };
-
-        this._cachedContent = undefined;
-
-        if (this.listeners.size > 0) {
-            const content = this.getCurrentContent();
-            for (const listener of this.listeners) {
-                listener(content);
-            }
-        }
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
 
         return true;
     }
@@ -269,10 +263,8 @@ export class CoValueCore {
         const nTxBefore = this.sessions[sessionID]?.transactions.length ?? 0;
 
         // const beforeHash = performance.now();
-        const { expectedNewHash, newStreamingHash } = await this.expectedNewHashAfterAsync(
-            sessionID,
-            newTransactions
-        );
+        const { expectedNewHash, newStreamingHash } =
+            await this.expectedNewHashAfterAsync(sessionID, newTransactions);
         // const afterHash = performance.now();
         // console.log(
         //     "Hashing took",
@@ -283,7 +275,7 @@ export class CoValueCore {
 
         if (nTxAfter !== nTxBefore) {
             const newTransactionLengthBefore = newTransactions.length;
-            newTransactions = newTransactions.slice((nTxAfter - nTxBefore));
+            newTransactions = newTransactions.slice(nTxAfter - nTxBefore);
             console.warn("Transactions changed while async hashing", {
                 nTxBefore,
                 nTxAfter,
@@ -303,7 +295,8 @@ export class CoValueCore {
         // const beforeVerify = performance.now();
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
-                "Invalid signature",
+                "Invalid signature in",
+                this.id,
                 newSignature,
                 expectedNewHash,
                 signerID
@@ -316,15 +309,61 @@ export class CoValueCore {
         //     afterVerify - beforeVerify
         // );
 
-        const transactions = this.sessions[sessionID]?.transactions ?? [];
+        this.doAddTransactions(
+            sessionID,
+            newTransactions,
+            newSignature,
+            expectedNewHash,
+            newStreamingHash
+        );
 
+        return true;
+    }
+
+    private doAddTransactions(
+        sessionID: SessionID,
+        newTransactions: Transaction[],
+        newSignature: Signature,
+        expectedNewHash: Hash,
+        newStreamingHash: StreamingHash
+    ) {
+        const transactions = this.sessions[sessionID]?.transactions ?? [];
         transactions.push(...newTransactions);
 
+        const signatureAfter = this.sessions[sessionID]?.signatureAfter ?? {};
+
+        const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
+            (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
+            -1
+        );
+
+        const sizeOfTxsSinceLastInbetweenSignature = transactions
+            .slice(lastInbetweenSignatureIdx + 1)
+            .reduce(
+                (sum, tx) =>
+                    sum +
+                    (tx.privacy === "private"
+                        ? tx.encryptedChanges.length
+                        : tx.changes.length),
+                0
+            );
+
+        if (sizeOfTxsSinceLastInbetweenSignature > 100 * 1024) {
+            // console.log(
+            //     "Saving inbetween signature for tx ",
+            //     sessionID,
+            //     transactions.length - 1,
+            //     sizeOfTxsSinceLastInbetweenSignature
+            // );
+            signatureAfter[transactions.length - 1] = newSignature;
+        }
+
         this._sessions[sessionID] = {
             transactions,
             lastHash: expectedNewHash,
             streamingHash: newStreamingHash,
             lastSignature: newSignature,
+            signatureAfter: signatureAfter,
         };
 
         this._cachedContent = undefined;
@@ -335,8 +374,6 @@ export class CoValueCore {
                 listener(content);
             }
         }
-
-        return true;
     }
 
     subscribe(listener: (content?: CoValueImpl) => void): () => void {
@@ -376,10 +413,10 @@ export class CoValueCore {
             new StreamingHash();
         let before = performance.now();
         for (const transaction of newTransactions) {
-            streamingHash.update(transaction)
+            streamingHash.update(transaction);
             const after = performance.now();
             if (after - before > 1) {
-                console.log("Hashing blocked for", after - before);
+                // console.log("Hashing blocked for", after - before);
                 await new Promise((resolve) => setTimeout(resolve, 0));
                 before = performance.now();
             }
@@ -415,7 +452,7 @@ export class CoValueCore {
                 tx: this.nextTransactionID(),
             });
 
-            this._decryptionCache[encrypted] = changes;
+            this._decryptionCache[encrypted] = stableStringify(changes);
 
             transaction = {
                 privacy: "private",
@@ -427,7 +464,7 @@ export class CoValueCore {
             transaction = {
                 privacy: "trusting",
                 madeAt,
-                changes,
+                changes: stableStringify(changes),
             };
         }
 
@@ -497,10 +534,11 @@ export class CoValueCore {
                     if (!readKey) {
                         return undefined;
                     } else {
-                        let decrytedChanges = this._decryptionCache[tx.encryptedChanges];
+                        let decrytedChanges =
+                            this._decryptionCache[tx.encryptedChanges];
 
                         if (!decrytedChanges) {
-                            decrytedChanges = decryptForTransaction(
+                            decrytedChanges = decryptRawForTransaction(
                                 tx.encryptedChanges,
                                 readKey,
                                 {
@@ -508,7 +546,8 @@ export class CoValueCore {
                                     tx: txID,
                                 }
                             );
-                            this._decryptionCache[tx.encryptedChanges] = decrytedChanges;
+                            this._decryptionCache[tx.encryptedChanges] =
+                                decrytedChanges;
                         }
 
                         if (!decrytedChanges) {
@@ -680,47 +719,95 @@ export class CoValueCore {
 
     newContentSince(
         knownState: CoValueKnownState | undefined
-    ): NewContentMessage | undefined {
-        const newContent: NewContentMessage = {
+    ): NewContentMessage[] | undefined {
+        let currentPiece: NewContentMessage = {
             action: "content",
             id: this.id,
             header: knownState?.header ? undefined : this.header,
-            new: Object.fromEntries(
-                Object.entries(this.sessions)
-                    .map(([sessionID, log]) => {
-                        const newTransactions = log.transactions.slice(
-                            knownState?.sessions[sessionID as SessionID] || 0
-                        );
+            new: {},
+        };
 
-                        if (
-                            newTransactions.length === 0 ||
-                            !log.lastHash ||
-                            !log.lastSignature
-                        ) {
-                            return undefined;
-                        }
+        const pieces = [currentPiece];
 
-                        return [
-                            sessionID,
-                            {
-                                after:
-                                    knownState?.sessions[
-                                        sessionID as SessionID
-                                    ] || 0,
-                                newTransactions,
-                                lastSignature: log.lastSignature,
-                            },
-                        ];
-                    })
-                    .filter((x): x is Exclude<typeof x, undefined> => !!x)
-            ),
+        const sentState: CoValueKnownState["sessions"] = {
+            ...knownState?.sessions,
         };
 
-        if (!newContent.header && Object.keys(newContent.new).length === 0) {
+        let newTxsWereAdded = true;
+        let pieceSize = 0;
+        while (newTxsWereAdded) {
+            newTxsWereAdded = false;
+
+            for (const [sessionID, log] of Object.entries(this.sessions) as [
+                SessionID,
+                SessionLog
+            ][]) {
+                const nextKnownSignatureIdx = Object.keys(log.signatureAfter)
+                    .map(Number)
+                    .sort((a, b) => a - b)
+                    .find((idx) => idx >= (sentState[sessionID] ?? -1));
+
+                const txsToAdd = log.transactions.slice(
+                    sentState[sessionID] ?? 0,
+                    nextKnownSignatureIdx === undefined
+                        ? undefined
+                        : nextKnownSignatureIdx + 1
+                );
+
+                if (txsToAdd.length === 0) continue;
+
+                newTxsWereAdded = true;
+
+                const oldPieceSize = pieceSize;
+                pieceSize += txsToAdd.reduce(
+                    (sum, tx) =>
+                        sum +
+                        (tx.privacy === "private"
+                            ? tx.encryptedChanges.length
+                            : tx.changes.length),
+                    0
+                );
+
+                if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
+                    currentPiece = {
+                        action: "content",
+                        id: this.id,
+                        header: undefined,
+                        new: {},
+                    };
+                    pieces.push(currentPiece);
+                    pieceSize = pieceSize - oldPieceSize;
+                }
+
+                let sessionEntry = currentPiece.new[sessionID];
+                if (!sessionEntry) {
+                    sessionEntry = {
+                        after: sentState[sessionID] ?? 0,
+                        newTransactions: [],
+                        lastSignature: "WILL_BE_REPLACED" as Signature
+                    };
+                    currentPiece.new[sessionID] = sessionEntry;
+                }
+
+                sessionEntry.newTransactions.push(...txsToAdd);
+                sessionEntry.lastSignature = nextKnownSignatureIdx === undefined
+                ? log.lastSignature!
+                : log.signatureAfter[nextKnownSignatureIdx]!
+
+                sentState[sessionID] =
+                    (sentState[sessionID] || 0) + txsToAdd.length;
+            }
+        }
+
+        const piecesWithContent = pieces.filter(
+            (piece) => Object.keys(piece.new).length > 0 || piece.header
+        );
+
+        if (piecesWithContent.length === 0) {
             return undefined;
         }
 
-        return newContent;
+        return piecesWithContent;
     }
 
     getDependedOnCoValues(): RawCoID[] {

package/src/coValues/coList.ts

@@ -4,6 +4,7 @@ import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { SessionID, TransactionID } from "../ids.js";
 import { Group } from "../group.js";
 import { AccountID, isAccountID } from "../account.js";
+import { parseJSON } from "../jsonStringify.js";
 
 type OpID = TransactionID & { changeIdx: number };
 
@@ -98,7 +99,7 @@ export class CoList<T extends JsonValue, Meta extends JsonObject | null = null>
             changes,
             madeAt,
         } of this.core.getValidSortedTransactions()) {
-            for (const [changeIdx, changeUntyped] of changes.entries()) {
+            for (const [changeIdx, changeUntyped] of parseJSON(changes).entries()) {
                 const change = changeUntyped as ListOpPayload<T>;
 
                 if (change.op === "pre" || change.op === "app") {

package/src/coValues/coMap.ts

@@ -4,15 +4,16 @@ import { CoID, ReadableCoValue, WriteableCoValue } from '../coValue.js';
 import { CoValueCore, accountOrAgentIDfromSessionID } from '../coValueCore.js';
 import { AccountID, isAccountID } from '../account.js';
 import { Group } from '../group.js';
+import { parseJSON } from '../jsonStringify.js';
 
-type MapOp<K extends string, V extends JsonValue> = {
+type MapOp<K extends string, V extends JsonValue | undefined> = {
     txID: TransactionID;
     madeAt: number;
     changeIdx: number;
 } & MapOpPayload<K, V>;
 // TODO: add after TransactionID[] for conflicts/ordering
 
-export type MapOpPayload<K extends string, V extends JsonValue> = {
+export type MapOpPayload<K extends string, V extends JsonValue | undefined> = {
     op: "set";
     key: K;
     value: V;
@@ -22,18 +23,16 @@ export type MapOpPayload<K extends string, V extends JsonValue> = {
     key: K;
 };
 
-export type MapK<M extends { [key: string]: JsonValue; }> = keyof M & string;
-export type MapV<M extends { [key: string]: JsonValue; }> = M[MapK<M>];
-export type MapM<M extends { [key: string]: JsonValue; }> = {
-    [KK in MapK<M>]: M[KK];
-}
+export type MapK<M extends { [key: string]: JsonValue | undefined; }> = keyof M & string;
+export type MapV<M extends { [key: string]: JsonValue | undefined; }> = M[MapK<M>];
+
 
 /** A collaborative map with precise shape `M` and optional static metadata `Meta` */
 export class CoMap<
-    M extends { [key: string]: JsonValue; },
+    M extends { [key: string]: JsonValue | undefined; },
     Meta extends JsonObject | null = null,
 > implements ReadableCoValue {
-    id: CoID<CoMap<MapM<M>, Meta>>;
+    id: CoID<CoMap<M, Meta>>;
     type = "comap" as const;
     core: CoValueCore;
     /** @internal */
@@ -43,7 +42,7 @@ export class CoMap<
 
     /** @internal */
     constructor(core: CoValueCore) {
-        this.id = core.id as CoID<CoMap<MapM<M>, Meta>>;
+        this.id = core.id as CoID<CoMap<M, Meta>>;
         this.core = core;
         this.ops = {};
 
@@ -64,7 +63,7 @@ export class CoMap<
 
         for (const { txID, changes, madeAt } of this.core.getValidSortedTransactions()) {
             for (const [changeIdx, changeUntyped] of (
-                changes
+                parseJSON(changes)
             ).entries()) {
                 const change = changeUntyped as MapOpPayload<MapK<M>, MapV<M>>;
                 let entries = this.ops[change.key];
@@ -207,7 +206,7 @@ export class CoMap<
 }
 
 export class WriteableCoMap<
-    M extends { [key: string]: JsonValue; },
+    M extends { [key: string]: JsonValue | undefined; },
     Meta extends JsonObject | null = null,
 > extends CoMap<M, Meta> implements WriteableCoValue {
     /** @internal */

package/src/coValues/coStream.ts

@@ -1,9 +1,12 @@
 import { JsonObject, JsonValue } from "../jsonValue.js";
 import { CoID, ReadableCoValue, WriteableCoValue } from "../coValue.js";
-import { CoValueCore } from "../coValueCore.js";
+import { CoValueCore, accountOrAgentIDfromSessionID } from "../coValueCore.js";
 import { Group } from "../group.js";
 import { SessionID } from "../ids.js";
 import { base64URLtoBytes, bytesToBase64url } from "../base64url.js";
+import { AccountID } from "../index.js";
+import { isAccountID } from "../account.js";
+import { parseJSON } from "../jsonStringify.js";
 
 export type BinaryChunkInfo = {
     mimeType: string;
@@ -40,7 +43,7 @@ export class CoStream<
     type = "costream" as const;
     core: CoValueCore;
     items: {
-        [key: SessionID]: T[];
+        [key: SessionID]: {item: T, madeAt: number}[];
     };
 
     constructor(core: CoValueCore) {
@@ -64,16 +67,17 @@ export class CoStream<
 
         for (const {
             txID,
+            madeAt,
             changes,
         } of this.core.getValidSortedTransactions()) {
-            for (const changeUntyped of changes) {
+            for (const changeUntyped of parseJSON(changes)) {
                 const change = changeUntyped as T;
                 let entries = this.items[txID.sessionID];
                 if (!entries) {
                     entries = [];
                     this.items[txID.sessionID] = entries;
                 }
-                entries.push(change);
+                entries.push({item: change, madeAt});
             }
         }
     }
@@ -87,13 +91,57 @@ export class CoStream<
             );
         }
 
-        return Object.values(this.items)[0];
+        return Object.values(this.items)[0]?.map(item => item.item);
+    }
+
+    getLastItemsPerAccount(): {[account: AccountID]: T | undefined} {
+        const result: {[account: AccountID]: {item: T, madeAt: number} | undefined} = {};
+
+        for (const [sessionID, items] of Object.entries(this.items)) {
+            const account = accountOrAgentIDfromSessionID(sessionID as SessionID);
+            if (!isAccountID(account)) continue;
+            if (items.length > 0) {
+                const lastItemOfSession = items[items.length - 1]!;
+                if (!result[account] || lastItemOfSession.madeAt > result[account]!.madeAt) {
+                    result[account] = lastItemOfSession;
+                }
+            }
+        }
+
+        return Object.fromEntries(Object.entries(result).map(([account, item]) =>
+            [account, item?.item]
+        ));
+    }
+
+    getLastItemFrom(account: AccountID): T | undefined {
+        let lastItem: {item: T, madeAt: number} | undefined;
+
+        for (const [sessionID, items] of Object.entries(this.items)) {
+            if (sessionID.startsWith(account)) {
+                if (items.length > 0) {
+                    const lastItemOfSession = items[items.length - 1]!;
+                    if (!lastItem || lastItemOfSession.madeAt > lastItem.madeAt) {
+                        lastItem = lastItemOfSession;
+                    }
+                }
+            }
+        }
+
+        return lastItem?.item;
+    }
+
+    getLastItemFromMe(): T | undefined {
+        const myAccountID = this.core.node.account.id;
+        if (!isAccountID(myAccountID)) return undefined;
+        return this.getLastItemFrom(myAccountID);
     }
 
     toJSON(): {
         [key: SessionID]: T[];
     } {
-        return this.items;
+        return Object.fromEntries(Object.entries(this.items).map(([sessionID, items]) =>
+            [sessionID, items.map(item => item.item)]
+        ));
     }
 
     subscribe(listener: (coMap: CoStream<T, Meta>) => void): () => void {
@@ -121,10 +169,10 @@ export class BinaryCoStream<
 {
     id!: CoID<BinaryCoStream<Meta>>;
 
-    getBinaryChunks():
+    getBinaryChunks(allowUnfinished?: boolean):
         | (BinaryChunkInfo & { chunks: Uint8Array[]; finished: boolean })
         | undefined {
-        const before = performance.now();
+        // const before = performance.now();
         const items = this.getSingleStream();
 
         if (!items) return;
@@ -136,10 +184,14 @@ export class BinaryCoStream<
             return;
         }
 
+        const end = items[items.length - 1];
+
+        if (end?.type !== "end" && !allowUnfinished) return;
+
         const chunks: Uint8Array[] = [];
 
         let finished = false;
-        let totalLength = 0;
+        // let totalLength = 0;
 
         for (const item of items.slice(1)) {
             if (item.type === "end") {
@@ -155,15 +207,15 @@ export class BinaryCoStream<
             const chunk = base64URLtoBytes(
                 item.chunk.slice(binary_U_prefixLength)
             );
-            totalLength += chunk.length;
+            // totalLength += chunk.length;
             chunks.push(chunk);
         }
 
-        const after = performance.now();
-        console.log(
-            "getBinaryChunks bandwidth in MB/s",
-            (1000 * totalLength) / (after - before) / (1024 * 1024)
-        );
+        // const after = performance.now();
+        // console.log(
+        //     "getBinaryChunks bandwidth in MB/s",
+        //     (1000 * totalLength) / (after - before) / (1024 * 1024)
+        // );
 
         return {
             mimeType: start.mimeType,
@@ -238,7 +290,7 @@ export class WriteableBinaryCoStream<
         chunk: Uint8Array,
         privacy: "private" | "trusting" = "private"
     ) {
-        const before = performance.now();
+        // const before = performance.now();
         this.push(
             {
                 type: "chunk",
@@ -246,11 +298,11 @@ export class WriteableBinaryCoStream<
             } satisfies BinaryStreamChunk,
             privacy
         );
-        const after = performance.now();
-        console.log(
-            "pushBinaryStreamChunk bandwidth in MB/s",
-            (1000 * chunk.length) / (after - before) / (1024 * 1024)
-        );
+        // const after = performance.now();
+        // console.log(
+        //     "pushBinaryStreamChunk bandwidth in MB/s",
+        //     (1000 * chunk.length) / (after - before) / (1024 * 1024)
+        // );
     }
 
     endBinaryStream(privacy: "private" | "trusting" = "private") {