cojson 0.7.0-alpha.29 → 0.7.0-alpha.36

package/src/storage/index.ts

@@ -0,0 +1,475 @@
+import {
+    ReadableStream,
+    WritableStream,
+    ReadableStreamDefaultReader,
+    WritableStreamDefaultWriter,
+} from "isomorphic-streams";
+import { Effect, Either, SynchronizedRef } from "effect";
+import { RawCoID } from "../ids.js";
+import { CoValueHeader, Transaction } from "../coValueCore.js";
+import { Signature } from "../crypto.js";
+import {
+    CoValueKnownState,
+    NewContentMessage,
+    Peer,
+    SyncMessage,
+} from "../sync.js";
+import { CoID, RawCoValue } from "../index.js";
+import { connectedPeers } from "../streamUtils.js";
+import {
+    chunkToKnownState,
+    contentSinceChunk,
+    mergeChunks,
+} from "./chunksAndKnownStates.js";
+import {
+    BlockFilename,
+    FSErr,
+    FileSystem,
+    WalEntry,
+    WalFilename,
+    readChunk,
+    readHeader,
+    textDecoder,
+    writeBlock,
+    writeToWal,
+} from "./FileSystem.js";
+export { FSErr, BlockFilename, WalFilename } from "./FileSystem.js";
+
+export type CoValueChunk = {
+    header?: CoValueHeader;
+    sessionEntries: {
+        [sessionID: string]: {
+            after: number;
+            lastSignature: Signature;
+            transactions: Transaction[];
+        }[];
+    };
+};
+
+export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
+    fromLocalNode!: ReadableStreamDefaultReader<SyncMessage>;
+    toLocalNode: WritableStreamDefaultWriter<SyncMessage>;
+    fs: FS;
+    currentWal: SynchronizedRef.SynchronizedRef<WH | undefined>;
+    coValues: SynchronizedRef.SynchronizedRef<{
+        [id: RawCoID]: CoValueChunk | undefined;
+    }>;
+    fileCache: string[] | undefined;
+    headerCache = new Map<
+        BlockFilename,
+        { [id: RawCoID]: { start: number; length: number } }
+    >();
+
+    constructor(
+        fs: FS,
+        fromLocalNode: ReadableStream<SyncMessage>,
+        toLocalNode: WritableStream<SyncMessage>
+    ) {
+        this.fs = fs;
+        this.fromLocalNode = fromLocalNode.getReader();
+        this.toLocalNode = toLocalNode.getWriter();
+        this.coValues = SynchronizedRef.unsafeMake({});
+        this.currentWal = SynchronizedRef.unsafeMake<WH | undefined>(undefined);
+
+        void Effect.runPromise(
+            Effect.gen(this, function* () {
+                let done = false;
+                while (!done) {
+                    const result = yield* Effect.promise(() =>
+                        this.fromLocalNode.read()
+                    );
+                    done = result.done;
+
+                    if (result.value) {
+                        if (result.value.action === "done") {
+                            continue;
+                        }
+
+                        if (result.value.action === "content") {
+                            yield* this.handleNewContent(result.value);
+                        } else {
+                            yield* this.sendNewContent(
+                                result.value.id,
+                                result.value,
+                                undefined
+                            );
+                        }
+                    }
+                }
+
+                return;
+            })
+        );
+
+        setTimeout(() => this.compact(), 20000);
+    }
+
+    sendNewContent(
+        id: RawCoID,
+        known: CoValueKnownState | undefined,
+        asDependencyOf: RawCoID | undefined
+    ): Effect.Effect<void, FSErr> {
+        return SynchronizedRef.updateEffect(this.coValues, (coValues) =>
+            this.sendNewContentInner(coValues, id, known, asDependencyOf)
+        );
+    }
+
+    private sendNewContentInner(
+        coValues: { [id: `co_z${string}`]: CoValueChunk | undefined },
+        id: RawCoID,
+        known: CoValueKnownState | undefined,
+        asDependencyOf: RawCoID | undefined
+    ): Effect.Effect<
+        { [id: `co_z${string}`]: CoValueChunk | undefined },
+        FSErr,
+        never
+    > {
+        return Effect.gen(this, function* () {
+            let coValue = coValues[id];
+
+            if (!coValue) {
+                coValue = yield* this.loadCoValue(id, this.fs);
+            }
+
+            if (!coValue) {
+                yield* Effect.promise(() =>
+                    this.toLocalNode.write({
+                        id: id,
+                        action: "known",
+                        header: false,
+                        sessions: {},
+                        asDependencyOf,
+                    })
+                );
+
+                return coValues;
+            }
+
+            if (
+                !known?.header &&
+                coValue.header?.ruleset.type === "ownedByGroup"
+            ) {
+                coValues = yield* this.sendNewContentInner(
+                    coValues,
+                    coValue.header.ruleset.group,
+                    undefined,
+                    asDependencyOf || id
+                );
+            } else if (
+                !known?.header &&
+                coValue.header?.ruleset.type === "group"
+            ) {
+                const dependedOnAccounts = new Set();
+                for (const session of Object.values(coValue.sessionEntries)) {
+                    for (const entry of session) {
+                        for (const tx of entry.transactions) {
+                            if (tx.privacy === "trusting") {
+                                const parsedChanges = JSON.parse(tx.changes);
+                                for (const change of parsedChanges) {
+                                    if (
+                                        change.op === "set" &&
+                                        change.key.startsWith("co_")
+                                    ) {
+                                        dependedOnAccounts.add(change.key);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                for (const account of dependedOnAccounts) {
+                    coValues = yield* this.sendNewContentInner(
+                        coValues,
+                        account as CoID<RawCoValue>,
+                        undefined,
+                        asDependencyOf || id
+                    );
+                }
+            }
+
+            const newContentMessages = contentSinceChunk(
+                id,
+                coValue,
+                known
+            ).map((message) => ({ ...message, asDependencyOf }));
+
+            const ourKnown: CoValueKnownState = chunkToKnownState(id, coValue);
+
+            yield* Effect.promise(() =>
+                this.toLocalNode.write({
+                    action: "known",
+                    ...ourKnown,
+                    asDependencyOf,
+                })
+            );
+
+            for (const message of newContentMessages) {
+                if (Object.keys(message.new).length === 0) continue;
+                yield* Effect.promise(() => this.toLocalNode.write(message));
+            }
+
+            return { ...coValues, [id]: coValue };
+        });
+    }
+
+    withWAL(
+        handler: (wal: WH) => Effect.Effect<void, FSErr>
+    ): Effect.Effect<void, FSErr> {
+        return SynchronizedRef.updateEffect(this.currentWal, (wal) =>
+            Effect.gen(this, function* () {
+                let newWal = wal;
+                if (!newWal) {
+                    newWal = yield* this.fs.createFile(
+                        `wal-${new Date().toISOString()}-${Math.random()
+                            .toString(36)
+                            .slice(2)}.jsonl`
+                    );
+                }
+                yield* handler(newWal);
+                return newWal;
+            })
+        );
+    }
+
+    handleNewContent(
+        newContent: NewContentMessage
+    ): Effect.Effect<void, FSErr> {
+        return SynchronizedRef.updateEffect(this.coValues, (coValues) =>
+            Effect.gen(this, function* () {
+                const coValue = coValues[newContent.id];
+
+                const newContentAsChunk: CoValueChunk = {
+                    header: newContent.header,
+                    sessionEntries: Object.fromEntries(
+                        Object.entries(newContent.new).map(
+                            ([sessionID, newInSession]) => [
+                                sessionID,
+                                [
+                                    {
+                                        after: newInSession.after,
+                                        lastSignature:
+                                            newInSession.lastSignature,
+                                        transactions:
+                                            newInSession.newTransactions,
+                                    },
+                                ],
+                            ]
+                        )
+                    ),
+                };
+
+                if (!coValue) {
+                    if (newContent.header) {
+                        console.log("Creating in WAL", newContent.id);
+                        yield* this.withWAL((wal) =>
+                            writeToWal(
+                                wal,
+                                this.fs,
+                                newContent.id,
+                                newContentAsChunk
+                            )
+                        );
+
+                        return {
+                            ...coValues,
+                            [newContent.id]: newContentAsChunk,
+                        };
+                    } else {
+                        // yield*
+                        //     Effect.promise(() =>
+                        //         this.toLocalNode.write({
+                        //             action: "known",
+                        //             id: newContent.id,
+                        //             header: false,
+                        //             sessions: {},
+                        //             isCorrection: true,
+                        //         })
+                        //     )
+                        // );
+                        console.warn(
+                            "Incontiguous incoming update for " + newContent.id
+                        );
+                        return coValues;
+                    }
+                } else {
+                    const merged = mergeChunks(coValue, newContentAsChunk);
+                    if (Either.isRight(merged)) {
+                        yield* Effect.logWarning(
+                            "Non-contigous new content for " + newContent.id
+                        );
+
+                        // yield* Effect.promise(() =>
+                        //     this.toLocalNode.write({
+                        //         action: "known",
+                        //         ...chunkToKnownState(newContent.id, coValue),
+                        //         isCorrection: true,
+                        //     })
+                        // );
+
+                        return coValues;
+                    } else {
+                        console.log("Appending to WAL", newContent.id);
+                        yield* this.withWAL((wal) =>
+                            writeToWal(
+                                wal,
+                                this.fs,
+                                newContent.id,
+                                newContentAsChunk
+                            )
+                        );
+
+                        return { ...coValues, [newContent.id]: merged.left };
+                    }
+                }
+            })
+        );
+    }
+
+    loadCoValue<WH, RH, FS extends FileSystem<WH, RH>>(
+        id: RawCoID,
+        fs: FS
+    ): Effect.Effect<CoValueChunk | undefined, FSErr> {
+        // return _loadChunkFromWal(id, fs);
+        return Effect.gen(this, function* () {
+            const files = this.fileCache || (yield* fs.listFiles());
+            this.fileCache = files;
+            const blockFiles = files.filter((name) =>
+                name.startsWith("hash_")
+            ) as BlockFilename[];
+
+            for (const blockFile of blockFiles) {
+                let cachedHeader:
+                    | { [id: RawCoID]: { start: number; length: number } }
+                    | undefined = this.headerCache.get(blockFile);
+
+                const { handle, size } = yield* fs.openToRead(blockFile);
+
+                if (!cachedHeader) {
+                    cachedHeader = {};
+                    const header = yield* readHeader(blockFile, handle, size, fs);
+                    for (const entry of header) {
+                        cachedHeader[entry.id] = {
+                            start: entry.start,
+                            length: entry.length,
+                        };
+                    }
+
+                    this.headerCache.set(blockFile, cachedHeader);
+                }
+                const headerEntry = cachedHeader[id];
+
+                let result;
+                if (headerEntry) {
+                    result = yield* readChunk(handle, headerEntry, fs);
+
+                }
+
+                yield* fs.close(handle);
+
+                return result
+            }
+
+            return undefined;
+        });
+    }
+
+    async compact() {
+        await Effect.runPromise(
+            Effect.gen(this, function* () {
+                const fileNames = yield* this.fs.listFiles();
+
+                const walFiles = fileNames.filter((name) =>
+                    name.startsWith("wal-")
+                ) as WalFilename[];
+                walFiles.sort();
+
+                const coValues = new Map<RawCoID, CoValueChunk>();
+
+                console.log("Compacting WAL files", walFiles);
+                if (walFiles.length === 0) return;
+
+                yield* SynchronizedRef.updateEffect(this.currentWal, (wal) =>
+                    Effect.gen(this, function* () {
+                        if (wal) {
+                            yield* this.fs.close(wal);
+                        }
+                        return undefined;
+                    })
+                );
+
+                for (const fileName of walFiles) {
+                    const { handle, size } =
+                        yield* this.fs.openToRead(fileName);
+                    if (size === 0) {
+                        yield* this.fs.close(handle);
+                        continue
+                    }
+                    const bytes = yield* this.fs.read(handle, 0, size);
+
+                    const decoded = textDecoder.decode(bytes);
+                    const lines = decoded.split("\n");
+
+                    for (const line of lines) {
+                        if (line.length === 0) continue;
+                        const chunk = JSON.parse(line) as WalEntry;
+
+                        const existingChunk = coValues.get(chunk.id);
+
+                        if (existingChunk) {
+                            const merged = mergeChunks(existingChunk, chunk);
+                            if (Either.isRight(merged)) {
+                                console.warn(
+                                    "Non-contigous chunks in " +
+                                        chunk.id +
+                                        ", " +
+                                        fileName,
+                                    existingChunk,
+                                    chunk
+                                );
+                            } else {
+                                coValues.set(chunk.id, merged.left);
+                            }
+                        } else {
+                            coValues.set(chunk.id, chunk);
+                        }
+                    }
+
+                    yield* this.fs.close(handle);
+                }
+
+                yield* writeBlock(coValues, 0, this.fs);
+                for (const walFile of walFiles) {
+                    yield* this.fs.removeFile(walFile);
+                }
+                this.fileCache = undefined;
+            })
+        );
+
+        setTimeout(() => this.compact(), 5000);
+    }
+
+    static asPeer<WH, RH, FS extends FileSystem<WH, RH>>({
+        fs,
+        trace,
+        localNodeName = "local",
+    }: {
+        fs: FS;
+        trace?: boolean;
+        localNodeName?: string;
+    }): Peer {
+        const [localNodeAsPeer, storageAsPeer] = connectedPeers(
+            localNodeName,
+            "storage",
+            {
+                peer1role: "client",
+                peer2role: "server",
+                trace,
+            }
+        );
+
+        new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
+
+        // return { ...storageAsPeer, priority: 200 };
+        return storageAsPeer;
+    }
+}

package/src/sync.ts

@@ -151,25 +151,25 @@ export class SyncManager {
                 throw new Error("Expected firstPeerState to be waiting " + id);
             }
             await new Promise<void>((resolve) => {
-                const timeout = setTimeout(() => {
-                    if (this.local.coValues[id]?.state === "loading") {
-                        // console.warn(
-                        //     "Timeout waiting for peer to load",
-                        //     id,
-                        //     "from",
-                        //     peer.id,
-                        //     "and it hasn't loaded from other peers yet"
-                        // );
-                    }
-                    resolve();
-                }, 1000);
+                // const timeout = setTimeout(() => {
+                //     if (this.local.coValues[id]?.state === "loading") {
+                //         console.warn(
+                //             "Timeout waiting for peer to load",
+                //             id,
+                //             "from",
+                //             peer.id,
+                //             "and it hasn't loaded from other peers yet"
+                //         );
+                //     }
+                //     resolve();
+                // }, 1000);
                 firstStateEntry.done
                     .then(() => {
-                        clearTimeout(timeout);
+                        // clearTimeout(timeout);
                         resolve();
                     })
                     .catch((e) => {
-                        clearTimeout(timeout);
+                        // clearTimeout(timeout);
                         console.error(
                             "Error waiting for peer to load",
                             id,
@@ -688,14 +688,14 @@ export class SyncManager {
             return this.requestedSyncs[coValue.id]!.done;
         } else {
             const done = new Promise<void>((resolve) => {
-                setTimeout(async () => {
+                queueMicrotask(async () => {
                     delete this.requestedSyncs[coValue.id];
                     // if (entry.nRequestsThisTick >= 2) {
                     //     console.log("Syncing", coValue.id, "for", entry.nRequestsThisTick, "requests");
                     // }
                     await this.actuallySyncCoValue(coValue);
                     resolve();
-                }, 0);
+                });
             });
             const entry = {
                 done,
@@ -707,14 +707,14 @@ export class SyncManager {
     }
 
     async actuallySyncCoValue(coValue: CoValueCore) {
-        let blockingSince = performance.now();
+        // let blockingSince = performance.now();
         for (const peer of this.peersInPriorityOrder()) {
-            if (performance.now() - blockingSince > 5) {
-                await new Promise<void>((resolve) => {
-                    setTimeout(resolve, 0);
-                });
-                blockingSince = performance.now();
-            }
+            // if (performance.now() - blockingSince > 5) {
+            //     await new Promise<void>((resolve) => {
+            //         setTimeout(resolve, 0);
+            //     });
+            //     blockingSince = performance.now();
+            // }
             const optimisticKnownState = peer.optimisticKnownStates[coValue.id];
 
             if (optimisticKnownState) {

package/src/tests/coList.test.ts

@@ -0,0 +1,100 @@
+import { expect, test, beforeEach } from "vitest";
+import { expectList, expectMap, expectStream } from "../coValue.js";
+import { RawBinaryCoStream } from "../coValues/coStream.js";
+import { createdNowUnique } from "../crypto.js";
+import { MAX_RECOMMENDED_TX_SIZE, cojsonReady } from "../index.js";
+import { LocalNode } from "../localNode.js";
+import { accountOrAgentIDfromSessionID } from "../typeUtils/accountOrAgentIDfromSessionID.js";
+import { randomAnonymousAccountAndSessionID } from "./testUtils.js";
+
+import { webcrypto } from "node:crypto";
+if (!("crypto" in globalThis)) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    (globalThis as any).crypto = webcrypto;
+}
+
+beforeEach(async () => {
+    await cojsonReady;
+});
+
+test("Empty CoList works", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = expectList(coValue.getCurrentContent());
+
+    expect(content.type).toEqual("colist");
+    expect(content.toJSON()).toEqual([]);
+});
+
+test("Can append, prepend, delete and replace items in CoList", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = expectList(coValue.getCurrentContent());
+
+    content.append("hello", 0, "trusting");
+    expect(content.toJSON()).toEqual(["hello"]);
+    content.append("world", 0, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "world"]);
+    content.prepend("beautiful", 1, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "beautiful", "world"]);
+    content.prepend("hooray", 3, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "beautiful", "world", "hooray"]);
+    content.replace(2, "universe", "trusting");
+    expect(content.toJSON()).toEqual(["hello", "beautiful", "universe", "hooray"]);
+    content.delete(2, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "beautiful", "hooray"]);
+});
+
+test("Push is equivalent to append after last item", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = expectList(coValue.getCurrentContent());
+
+    expect(content.type).toEqual("colist");
+
+    content.append("hello", 0, "trusting");
+    expect(content.toJSON()).toEqual(["hello"]);
+    content.append("world", undefined, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "world"]);
+    content.append("hooray", undefined, "trusting");
+    expect(content.toJSON()).toEqual(["hello", "world", "hooray"]);
+});
+
+test("Can push into empty list", () => {
+    const node = new LocalNode(...randomAnonymousAccountAndSessionID());
+
+    const coValue = node.createCoValue({
+        type: "colist",
+        ruleset: { type: "unsafeAllowAll" },
+        meta: null,
+        ...createdNowUnique(),
+    });
+
+    const content = expectList(coValue.getCurrentContent());
+
+    expect(content.type).toEqual("colist");
+
+    content.append("hello", undefined, "trusting");
+    expect(content.toJSON()).toEqual(["hello"]);
+});