cojson 0.7.0-alpha.29 → 0.7.0-alpha.36

package/src/coValueCore.ts

@@ -250,7 +250,8 @@ export class CoValueCore {
             newTransactions,
             newSignature,
             expectedNewHash,
-            newStreamingHash
+            newStreamingHash,
+            "immediate"
         );
 
         return true;
@@ -287,7 +288,8 @@ export class CoValueCore {
             return false;
         }
 
-        const nTxBefore = this.sessionLogs.get(sessionID)?.transactions.length ?? 0;
+        const nTxBefore =
+            this.sessionLogs.get(sessionID)?.transactions.length ?? 0;
 
         // const beforeHash = performance.now();
         const { expectedNewHash, newStreamingHash } =
@@ -298,7 +300,8 @@ export class CoValueCore {
         //     afterHash - beforeHash
         // );
 
-        const nTxAfter = this.sessionLogs.get(sessionID)?.transactions.length ?? 0;
+        const nTxAfter =
+            this.sessionLogs.get(sessionID)?.transactions.length ?? 0;
 
         if (nTxAfter !== nTxBefore) {
             const newTransactionLengthBefore = newTransactions.length;
@@ -320,7 +323,7 @@ export class CoValueCore {
             return false;
         }
 
-        // const beforeVerify = performance.now();
+        performance.mark("verifyStart" + this.id);
         if (!verify(newSignature, expectedNewHash, signerID)) {
             console.warn(
                 "Invalid signature in",
@@ -332,18 +335,20 @@ export class CoValueCore {
             resolveDone();
             return false;
         }
-        // const afterVerify = performance.now();
-        // console.log(
-        //     "Verify took",
-        //     afterVerify - beforeVerify
-        // );
+        performance.mark("verifyEnd" + this.id);
+        performance.measure(
+            "verify" + this.id,
+            "verifyStart" + this.id,
+            "verifyEnd" + this.id
+        );
 
         this.doAddTransactions(
             sessionID,
             newTransactions,
             newSignature,
             expectedNewHash,
-            newStreamingHash
+            newStreamingHash,
+            "deferred"
         );
 
         resolveDone();
@@ -355,12 +360,15 @@ export class CoValueCore {
         newTransactions: Transaction[],
         newSignature: Signature,
         expectedNewHash: Hash,
-        newStreamingHash: StreamingHash
+        newStreamingHash: StreamingHash,
+        notifyMode: "immediate" | "deferred"
     ) {
-        const transactions = this.sessionLogs.get(sessionID)?.transactions ?? [];
+        const transactions =
+            this.sessionLogs.get(sessionID)?.transactions ?? [];
         transactions.push(...newTransactions);
 
-        const signatureAfter = this.sessionLogs.get(sessionID)?.signatureAfter ?? {};
+        const signatureAfter =
+            this.sessionLogs.get(sessionID)?.signatureAfter ?? {};
 
         const lastInbetweenSignatureIdx = Object.keys(signatureAfter).reduce(
             (max, idx) => (parseInt(idx) > max ? parseInt(idx) : max),
@@ -402,13 +410,33 @@ export class CoValueCore {
         this._cachedNewContentSinceEmpty = undefined;
 
         if (this.listeners.size > 0) {
-            const content = this.getCurrentContent();
-            for (const listener of this.listeners) {
-                listener(content);
+            if (notifyMode === "immediate") {
+                const content = this.getCurrentContent();
+                for (const listener of this.listeners) {
+                    listener(content);
+                }
+            } else {
+                if (!this.nextDeferredNotify) {
+                    this.nextDeferredNotify = new Promise((resolve) => {
+                        setTimeout(() => {
+                            this.nextDeferredNotify = undefined;
+                            this.deferredUpdates = 0;
+                            const content = this.getCurrentContent();
+                            for (const listener of this.listeners) {
+                                listener(content);
+                            }
+                            resolve();
+                        }, 0);
+                    });
+                }
+                this.deferredUpdates++;
             }
         }
     }
 
+    deferredUpdates = 0;
+    nextDeferredNotify: Promise<void> | undefined;
+
     subscribe(listener: (content?: RawCoValue) => void): () => void {
         this.listeners.add(listener);
         listener(this.getCurrentContent());
@@ -533,7 +561,9 @@ export class CoValueCore {
         return success;
     }
 
-    getCurrentContent(options?: { ignorePrivateTransactions: true }): RawCoValue {
+    getCurrentContent(options?: {
+        ignorePrivateTransactions: true;
+    }): RawCoValue {
         if (!options?.ignorePrivateTransactions && this._cachedContent) {
             return this._cachedContent;
         }
@@ -781,7 +811,11 @@ export class CoValueCore {
 
         let sessionsTodoAgain: Set<SessionID> | undefined | "first" = "first";
 
-        while (sessionsTodoAgain === "first" || (sessionsTodoAgain?.size || 0 > 0)) {
+        while (
+            sessionsTodoAgain === "first" ||
+            sessionsTodoAgain?.size ||
+            0 > 0
+        ) {
             if (sessionsTodoAgain === "first") {
                 sessionsTodoAgain = undefined;
             }
@@ -798,10 +832,12 @@ export class CoValueCore {
                     sentStateForSessionID
                 );
 
-                const firstNewTxIdx = sentStateForSessionID ?? knownStateForSessionID ?? 0;
-                const afterLastNewTxIdx = nextKnownSignatureIdx === undefined
-                ? log.transactions.length
-                : nextKnownSignatureIdx + 1;
+                const firstNewTxIdx =
+                    sentStateForSessionID ?? knownStateForSessionID ?? 0;
+                const afterLastNewTxIdx =
+                    nextKnownSignatureIdx === undefined
+                        ? log.transactions.length
+                        : nextKnownSignatureIdx + 1;
 
                 const nNewTx = Math.max(0, afterLastNewTxIdx - firstNewTxIdx);
 
@@ -818,11 +854,16 @@ export class CoValueCore {
                 }
 
                 const oldPieceSize = pieceSize;
-                for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
+                for (
+                    let txIdx = firstNewTxIdx;
+                    txIdx < afterLastNewTxIdx;
+                    txIdx++
+                ) {
                     const tx = log.transactions[txIdx]!;
-                    pieceSize += (tx.privacy === "private"
-                    ? tx.encryptedChanges.length
-                    : tx.changes.length);
+                    pieceSize +=
+                        tx.privacy === "private"
+                            ? tx.encryptedChanges.length
+                            : tx.changes.length;
                 }
 
                 if (pieceSize >= MAX_RECOMMENDED_TX_SIZE) {
@@ -839,26 +880,33 @@ export class CoValueCore {
                 let sessionEntry = currentPiece.new[sessionID];
                 if (!sessionEntry) {
                     sessionEntry = {
-                        after: sentStateForSessionID ?? knownStateForSessionID ?? 0,
+                        after:
+                            sentStateForSessionID ??
+                            knownStateForSessionID ??
+                            0,
                         newTransactions: [],
                         lastSignature: "WILL_BE_REPLACED" as Signature,
                     };
                     currentPiece.new[sessionID] = sessionEntry;
                 }
 
-                for (let txIdx = firstNewTxIdx; txIdx < afterLastNewTxIdx; txIdx++) {
+                for (
+                    let txIdx = firstNewTxIdx;
+                    txIdx < afterLastNewTxIdx;
+                    txIdx++
+                ) {
                     const tx = log.transactions[txIdx]!;
                     sessionEntry.newTransactions.push(tx);
                 }
 
-
                 sessionEntry.lastSignature =
                     nextKnownSignatureIdx === undefined
                         ? log.lastSignature!
                         : log.signatureAfter[nextKnownSignatureIdx]!;
 
                 sentState[sessionID] =
-                    (sentStateForSessionID ?? knownStateForSessionID ?? 0) + nNewTx;
+                    (sentStateForSessionID ?? knownStateForSessionID ?? 0) +
+                    nNewTx;
             }
         }
 
@@ -894,32 +942,35 @@ export class CoValueCore {
                   .keys()
                   .filter((k): k is AccountID => k.startsWith("co_"))
             : this.header.ruleset.type === "ownedByGroup"
-            ? [
-                  this.header.ruleset.group,
-                  ...new Set(
-                      [...this.sessionLogs.keys()]
-                          .map((sessionID) =>
-                              accountOrAgentIDfromSessionID(
-                                  sessionID as SessionID
-                              )
-                          )
-                          .filter(
-                              (session): session is AccountID =>
-                                  isAccountID(session) && session !== this.id
-                          )
-                  ),
-              ]
-            : [];
+              ? [
+                    this.header.ruleset.group,
+                    ...new Set(
+                        [...this.sessionLogs.keys()]
+                            .map((sessionID) =>
+                                accountOrAgentIDfromSessionID(
+                                    sessionID as SessionID
+                                )
+                            )
+                            .filter(
+                                (session): session is AccountID =>
+                                    isAccountID(session) && session !== this.id
+                            )
+                    ),
+                ]
+              : [];
     }
 }
 
 function getNextKnownSignatureIdx(
     log: SessionLog,
     knownStateForSessionID?: number,
-    sentStateForSessionID?: number,
+    sentStateForSessionID?: number
 ) {
     return Object.keys(log.signatureAfter)
         .map(Number)
         .sort((a, b) => a - b)
-        .find((idx) => idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1));
+        .find(
+            (idx) =>
+                idx >= (sentStateForSessionID ?? knownStateForSessionID ?? -1)
+        );
 }

package/src/crypto.ts

@@ -276,7 +276,7 @@ export class StreamingHash {
         this.state = fromClone || blake3Instance.init().save();
     }
 
-    update(value: JsonValue) {
+    update(value: JsonValue): Uint8Array {
         const encoded = textEncoder.encode(stableStringify(value));
         // const before = performance.now();
         this.state = blake3incrementalUpdateSLOW_WITH_DEVTOOLS(
@@ -285,6 +285,7 @@ export class StreamingHash {
         );
         // const after = performance.now();
         // console.log(`Hashing throughput in MB/s`, 1000 * (encoded.length / (after - before)) / (1024 * 1024));
+        return encoded;
     }
 
     digest(): Hash {

package/src/index.ts

@@ -20,6 +20,7 @@ import {
     secretSeedLength,
     shortHashLength,
     cryptoReady,
+    StreamingHash
 } from "./crypto.js";
 import { connectedPeers } from "./streamUtils.js";
 import { ControlledAgent, RawControlledAccount } from "./coValues/account.js";
@@ -56,6 +57,14 @@ import type * as Media from "./media.js";
 
 type Value = JsonValue | AnyRawCoValue;
 
+import {
+    LSMStorage,
+    FSErr,
+    BlockFilename,
+    WalFilename,
+} from "./storage/index.js";
+import { FileSystem } from "./storage/FileSystem.js";
+
 /** @hidden */
 export const cojsonInternals = {
     agentSecretFromBytes,
@@ -78,6 +87,7 @@ export const cojsonInternals = {
     isAccountID,
     accountHeaderForInitialAgentSecret,
     idforHeader,
+    StreamingHash
 };
 
 export {
@@ -113,7 +123,12 @@ export {
     AgentSecret,
     InviteSecret,
     SyncMessage,
-    isRawCoID
+    isRawCoID,
+    FileSystem,
+    LSMStorage,
+    FSErr,
+    BlockFilename,
+    WalFilename,
 };
 
 export type { Value };

package/src/localNode.ts

@@ -534,7 +534,7 @@ export class LocalNode {
             );
         }
 
-        return new RawAccount(coValue).currentAgentID();
+        return (coValue.getCurrentContent() as RawAccount).currentAgentID();
     }
 
     async resolveAccountAgentAsync(
@@ -569,7 +569,7 @@ export class LocalNode {
             );
         }
 
-        return new RawAccount(coValue).currentAgentID();
+        return (coValue.getCurrentContent() as RawAccount).currentAgentID();
     }
 
     /**

package/src/storage/FileSystem.ts

@@ -0,0 +1,151 @@
+import { Effect } from "effect";
+import { CoValueChunk } from "./index.js";
+import { RawCoID } from "../ids.js";
+import { StreamingHash } from "../crypto.js";
+
+export type BlockFilename =
+    `${string}-L${number}-H${number}.jsonl`;
+
+    export type BlockHeader = { id: RawCoID; start: number; length: number }[];
+
+export type WalEntry = { id: RawCoID } & CoValueChunk;
+
+export type WalFilename = `wal-${number}.jsonl`;
+
+export type FSErr = {
+    type: "fileSystemError";
+    error: Error;
+};
+
+export interface FileSystem<WriteHandle, ReadHandle> {
+    createFile(filename: string): Effect.Effect<WriteHandle, FSErr>;
+    append(handle: WriteHandle, data: Uint8Array): Effect.Effect<void, FSErr>;
+    close(
+        handle: ReadHandle | WriteHandle,
+    ): Effect.Effect<void, FSErr>;
+    closeAndRename(
+        handle: WriteHandle,
+        filename: BlockFilename
+    ): Effect.Effect<void, FSErr>;
+    openToRead(
+        filename: string
+    ): Effect.Effect<{ handle: ReadHandle; size: number; }, FSErr>;
+    read(
+        handle: ReadHandle,
+        offset: number,
+        length: number
+    ): Effect.Effect<Uint8Array, FSErr>;
+    listFiles(): Effect.Effect<string[], FSErr>;
+    removeFile(filename: BlockFilename | WalFilename): Effect.Effect<void, FSErr>;
+}
+
+export const textEncoder = new TextEncoder();
+export const textDecoder = new TextDecoder();
+
+export function readChunk<RH, FS extends FileSystem<unknown, RH>>(
+    handle: RH,
+    header: { start: number; length: number; },
+    fs: FS
+): Effect.Effect<CoValueChunk, FSErr> {
+    return Effect.gen(function* ($) {
+        const chunkBytes = yield* $(
+            fs.read(handle, header.start, header.length)
+        );
+
+        const chunk = JSON.parse(textDecoder.decode(chunkBytes));
+        return chunk;
+    });
+}
+
+export function readHeader<RH, FS extends FileSystem<unknown, RH>>(
+    filename: string,
+    handle: RH,
+    size: number,
+    fs: FS
+): Effect.Effect<BlockHeader, FSErr> {
+    return Effect.gen(function* ($) {
+
+
+        const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
+
+        const headerBytes = yield* $(
+            fs.read(handle, size - headerLength, headerLength)
+        );
+
+        const header = JSON.parse(textDecoder.decode(headerBytes));
+        return header;
+    });
+}
+
+export function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
+    chunks: Map<RawCoID, CoValueChunk>,
+    level: number,
+    fs: FS
+): Effect.Effect<void, FSErr> {
+    if (chunks.size === 0) {
+        return Effect.die(new Error("No chunks to write"));
+    }
+
+    return Effect.gen(function* ($) {
+        const blockHeader: BlockHeader = [];
+
+        let offset = 0;
+
+        const file = yield* $(
+            fs.createFile(
+                "wipBlock" +
+                Math.random().toString(36).substring(7) +
+                ".tmp.jsonl"
+            )
+        );
+        const hash = new StreamingHash();
+
+        const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) => id1.localeCompare(id2)
+        );
+
+        for (const [id, chunk] of chunksSortedById) {
+            const encodedBytes = hash.update(chunk);
+            const encodedBytesWithNewline = new Uint8Array(
+                encodedBytes.length + 1
+            );
+            encodedBytesWithNewline.set(encodedBytes);
+            encodedBytesWithNewline[encodedBytes.length] = 10;
+            yield* $(fs.append(file, encodedBytesWithNewline));
+            const length = encodedBytesWithNewline.length;
+            blockHeader.push({ id, start: offset, length });
+            offset += length;
+        }
+
+        const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
+        yield* $(fs.append(file, headerBytes));
+
+        console.log(
+            "full file",
+            yield* $(
+                fs.read(file as unknown as RH, 0, offset + headerBytes.length)
+            )
+        );
+
+        const filename: BlockFilename = `${hash.digest()}-L${level}-H${headerBytes.length}.jsonl`;
+        console.log("renaming to" + filename);
+        yield* $(fs.closeAndRename(file, filename));
+
+        console.log("Wrote block", filename, blockHeader);
+    });
+}
+
+export function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
+    handle: WH,
+    fs: FS,
+    id: RawCoID,
+    chunk: CoValueChunk
+): Effect.Effect<void, FSErr> {
+    return Effect.gen(function* ($) {
+        const walEntry: WalEntry = {
+            id,
+            ...chunk,
+        };
+        const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
+        yield* $(fs.append(handle, bytes));
+    });
+}

package/src/storage/chunksAndKnownStates.ts

@@ -0,0 +1,132 @@
+import { Either } from 'effect';
+import { RawCoID, SessionID } from '../ids.js';
+import { MAX_RECOMMENDED_TX_SIZE } from '../index.js';
+import { CoValueKnownState, NewContentMessage } from "../sync.js";
+import { CoValueChunk } from "./index.js";
+
+export function contentSinceChunk(
+    id: RawCoID,
+    chunk: CoValueChunk,
+    known?: CoValueKnownState
+): NewContentMessage[] {
+    const newContentPieces: NewContentMessage[] = [];
+
+    newContentPieces.push({
+        id: id,
+        action: "content",
+        header: known?.header ? undefined : chunk.header,
+        new: {},
+    });
+
+    for (const [sessionID, sessionsEntry] of Object.entries(
+        chunk.sessionEntries
+    )) {
+        for (const entry of sessionsEntry) {
+            const knownStart = known?.sessions[sessionID as SessionID] || 0;
+
+            if (entry.after + entry.transactions.length <= knownStart) {
+                continue;
+            }
+
+            const actuallyNewTransactions = entry.transactions.slice(
+                Math.max(0, knownStart - entry.after)
+            );
+
+            const newAfter = entry.after +
+                (actuallyNewTransactions.length - entry.transactions.length);
+
+            let newContentEntry = newContentPieces[0]?.new[sessionID as SessionID];
+
+            if (!newContentEntry) {
+                newContentEntry = {
+                    after: newAfter,
+                    lastSignature: entry.lastSignature,
+                    newTransactions: actuallyNewTransactions,
+                };
+                newContentPieces[0]!.new[sessionID as SessionID] =
+                    newContentEntry;
+            } else {
+                newContentEntry.newTransactions.push(
+                    ...actuallyNewTransactions
+                );
+                newContentEntry.lastSignature = entry.lastSignature;
+            }
+        }
+    }
+
+    return newContentPieces;
+}
+
+export function chunkToKnownState(id: RawCoID, chunk: CoValueChunk) {
+    const ourKnown: CoValueKnownState = {
+        id,
+        header: !!chunk.header,
+        sessions: {},
+    };
+
+    for (const [sessionID, sessionEntries] of Object.entries(
+        chunk.sessionEntries
+    )) {
+        for (const entry of sessionEntries) {
+            ourKnown.sessions[sessionID as SessionID] =
+                entry.after + entry.transactions.length;
+        }
+    }
+    return ourKnown;
+}
+
+export function mergeChunks(
+    chunkA: CoValueChunk,
+    chunkB: CoValueChunk
+): Either.Either<"nonContigous", CoValueChunk> {
+    const header = chunkA.header || chunkB.header;
+
+    const newSessions = { ...chunkA.sessionEntries };
+    for (const sessionID in chunkB.sessionEntries) {
+        // figure out if we can merge the chunks
+        const sessionEntriesA = chunkA.sessionEntries[sessionID];
+        const sessionEntriesB = chunkB.sessionEntries[sessionID]!;
+
+        if (!sessionEntriesA) {
+            newSessions[sessionID] = sessionEntriesB;
+            continue;
+        }
+
+        const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1]!;
+        const firstEntryOfB = sessionEntriesB[0]!;
+
+        if (lastEntryOfA.after + lastEntryOfA.transactions.length ===
+            firstEntryOfB.after) {
+            const newEntries = [];
+            let bytesSinceLastSignature = 0;
+            for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
+                const entryByteLength = entry.transactions.reduce(
+                    (sum, tx) => sum +
+                        (tx.privacy === "private"
+                            ? tx.encryptedChanges.length
+                            : tx.changes.length),
+                    0
+                );
+                if (newEntries.length === 0 ||
+                    bytesSinceLastSignature + entryByteLength >
+                    MAX_RECOMMENDED_TX_SIZE) {
+                    newEntries.push({
+                        after: entry.after,
+                        lastSignature: entry.lastSignature,
+                        transactions: entry.transactions,
+                    });
+                    bytesSinceLastSignature = 0;
+                } else {
+                    const lastNewEntry = newEntries[newEntries.length - 1]!;
+                    lastNewEntry.transactions.push(...entry.transactions);
+
+                    bytesSinceLastSignature += entry.transactions.length;
+                }
+            }
+        } else {
+            return Either.right("nonContigous" as const);
+        }
+    }
+
+    return Either.left({ header, sessionEntries: newSessions });
+}