cojson 0.7.17 → 0.7.23

package/src/storage/index.ts

@@ -1,4 +1,11 @@
-import { Effect, Either, Queue, Stream, SynchronizedRef } from "effect";
+import {
+    Effect,
+    Either,
+    Queue,
+    Stream,
+    SynchronizedRef,
+    Deferred,
+} from "effect";
 import { RawCoID } from "../ids.js";
 import { CoValueHeader, Transaction } from "../coValueCore.js";
 import { Signature } from "../crypto/crypto.js";
@@ -30,6 +37,8 @@ import {
 } from "./FileSystem.js";
 export type { FSErr, BlockFilename, WalFilename } from "./FileSystem.js";
 
+const MAX_N_LEVELS = 3;
+
 export type CoValueChunk = {
     header?: CoValueHeader;
     sessionEntries: {
@@ -51,6 +60,10 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         BlockFilename,
         { [id: RawCoID]: { start: number; length: number } }
     >();
+    blockFileHandles = new Map<
+        BlockFilename,
+        Deferred.Deferred<{ handle: RH; size: number }, FSErr>
+    >();
 
     constructor(
         public fs: FS,
@@ -192,7 +205,7 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                 let newWal = wal;
                 if (!newWal) {
                     newWal = yield* this.fs.createFile(
-                        `wal-${new Date().toISOString()}-${Math.random()
+                        `wal-${Date.now()}-${Math.random()
                             .toString(36)
                             .slice(2)}.jsonl`,
                     );
@@ -314,24 +327,63 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
         );
     }
 
-    loadCoValue<WH, RH, FS extends FileSystem<WH, RH>>(
+    getBlockHandle(
+        blockFile: BlockFilename,
+        fs: FS,
+    ): Effect.Effect<{ handle: RH; size: number }, FSErr> {
+        return Effect.gen(this, function* () {
+            let handleAndSize = this.blockFileHandles.get(blockFile);
+            if (!handleAndSize) {
+                handleAndSize = yield* Deferred.make<
+                    { handle: RH; size: number },
+                    FSErr
+                >();
+                this.blockFileHandles.set(blockFile, handleAndSize);
+                yield* Deferred.complete(
+                    handleAndSize,
+                    fs.openToRead(blockFile),
+                );
+            }
+
+            return yield* Deferred.await(handleAndSize);
+        });
+    }
+
+    loadCoValue(
         id: RawCoID,
         fs: FS,
     ): Effect.Effect<CoValueChunk | undefined, FSErr> {
-        // return _loadChunkFromWal(id, fs);
         return Effect.gen(this, function* () {
             const files = this.fileCache || (yield* fs.listFiles());
             this.fileCache = files;
-            const blockFiles = files.filter((name) =>
-                name.startsWith("hash_"),
-            ) as BlockFilename[];
+            const blockFiles = (
+                files.filter((name) => name.startsWith("L")) as BlockFilename[]
+            ).sort();
+
+            let result;
 
             for (const blockFile of blockFiles) {
                 let cachedHeader:
                     | { [id: RawCoID]: { start: number; length: number } }
                     | undefined = this.headerCache.get(blockFile);
 
-                const { handle, size } = yield* fs.openToRead(blockFile);
+                let handleAndSize = this.blockFileHandles.get(blockFile);
+                if (!handleAndSize) {
+                    handleAndSize = yield* Deferred.make<
+                        { handle: RH; size: number },
+                        FSErr
+                    >();
+                    this.blockFileHandles.set(blockFile, handleAndSize);
+                    yield* Deferred.complete(
+                        handleAndSize,
+                        fs.openToRead(blockFile),
+                    );
+                }
+
+                const { handle, size } = yield* this.getBlockHandle(
+                    blockFile,
+                    fs,
+                );
 
                 // console.log("Attempting to load", id, blockFile);
 
@@ -356,17 +408,29 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
 
                 // console.log("Header entry", id, headerEntry);
 
-                let result;
                 if (headerEntry) {
-                    result = yield* readChunk(handle, headerEntry, fs);
+                    const nextChunk = yield* readChunk(handle, headerEntry, fs);
+                    if (result) {
+                        const merged = mergeChunks(result, nextChunk);
+
+                        if (Either.isRight(merged)) {
+                            yield* Effect.logWarning(
+                                "Non-contigous chunks while loading " + id,
+                                result,
+                                nextChunk,
+                            );
+                        } else {
+                            result = merged.left;
+                        }
+                    } else {
+                        result = nextChunk;
+                    }
                 }
 
-                yield* fs.close(handle);
-
-                return result;
+                // yield* fs.close(handle);
             }
 
-            return undefined;
+            return result;
         });
     }
 
@@ -434,11 +498,150 @@ export class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
                     yield* this.fs.close(handle);
                 }
 
-                yield* writeBlock(coValues, 0, this.fs);
+                const highestBlockNumber = fileNames.reduce((acc, name) => {
+                    if (name.startsWith("L" + MAX_N_LEVELS)) {
+                        const num = parseInt(name.split("-")[1]!);
+                        if (num > acc) {
+                            return num;
+                        }
+                    }
+                    return acc;
+                }, 0);
+
+                console.log(
+                    [...coValues.keys()],
+                    fileNames,
+                    highestBlockNumber,
+                );
+
+                yield* writeBlock(
+                    coValues,
+                    MAX_N_LEVELS,
+                    highestBlockNumber + 1,
+                    this.fs,
+                );
+
                 for (const walFile of walFiles) {
                     yield* this.fs.removeFile(walFile);
                 }
                 this.fileCache = undefined;
+
+                const fileNames2 = yield* this.fs.listFiles();
+
+                const blockFiles = (
+                    fileNames2.filter((name) =>
+                        name.startsWith("L"),
+                    ) as BlockFilename[]
+                ).sort();
+
+                const blockFilesByLevelInOrder: {
+                    [level: number]: BlockFilename[];
+                } = {};
+
+                for (const blockFile of blockFiles) {
+                    const level = parseInt(blockFile.split("-")[0]!.slice(1));
+                    if (!blockFilesByLevelInOrder[level]) {
+                        blockFilesByLevelInOrder[level] = [];
+                    }
+                    blockFilesByLevelInOrder[level]!.push(blockFile);
+                }
+
+                console.log(blockFilesByLevelInOrder);
+
+                for (let level = MAX_N_LEVELS; level > 0; level--) {
+                    const nBlocksDesired = Math.pow(2, level);
+                    const blocksInLevel = blockFilesByLevelInOrder[level];
+
+                    if (
+                        blocksInLevel &&
+                        blocksInLevel.length > nBlocksDesired
+                    ) {
+                        yield* Effect.log("Compacting blocks in level", level, blocksInLevel);
+
+                        const coValues = new Map<RawCoID, CoValueChunk>();
+
+                        for (const blockFile of blocksInLevel) {
+                            const {
+                                handle,
+                                size,
+                            }: { handle: RH; size: number } =
+                                yield* this.getBlockHandle(blockFile, this.fs);
+
+                            if (size === 0) {
+                                continue;
+                            }
+                            const header = yield* readHeader(
+                                blockFile,
+                                handle,
+                                size,
+                                this.fs,
+                            );
+                            for (const entry of header) {
+                                const chunk = yield* readChunk(
+                                    handle,
+                                    entry,
+                                    this.fs,
+                                );
+
+                                const existingChunk = coValues.get(entry.id);
+
+                                if (existingChunk) {
+                                    const merged = mergeChunks(
+                                        existingChunk,
+                                        chunk,
+                                    );
+                                    if (Either.isRight(merged)) {
+                                        yield* Effect.logWarning(
+                                            "Non-contigous chunks in " +
+                                                entry.id +
+                                                ", " +
+                                                blockFile,
+                                            existingChunk,
+                                            chunk,
+                                        );
+                                    } else {
+                                        coValues.set(entry.id, merged.left);
+                                    }
+                                } else {
+                                    coValues.set(entry.id, chunk);
+                                }
+                            }
+                        }
+
+                        let levelBelow = blockFilesByLevelInOrder[level - 1];
+                        if (!levelBelow) {
+                            levelBelow = [];
+                            blockFilesByLevelInOrder[level - 1] = levelBelow;
+                        }
+
+                        const highestBlockNumberInLevelBelow =
+                            levelBelow.reduce((acc, name) => {
+                                const num = parseInt(name.split("-")[1]!);
+                                if (num > acc) {
+                                    return num;
+                                }
+                                return acc;
+                            }, 0);
+
+                        const newBlockName = yield* writeBlock(
+                            coValues,
+                            level - 1,
+                            highestBlockNumberInLevelBelow + 1,
+                            this.fs,
+                        );
+                        levelBelow.push(newBlockName);
+
+                        // delete blocks that went into this one
+                        for (const blockFile of blocksInLevel) {
+                            const handle = yield* this.getBlockHandle(
+                                blockFile,
+                                this.fs,
+                            );
+                            yield* this.fs.close(handle.handle);
+                            yield* this.fs.removeFile(blockFile);
+                        }
+                    }
+                }
             }),
         );
 

package/src/streamUtils.ts

@@ -47,18 +47,24 @@ export function newQueuePair(
         const queue = yield* Queue.unbounded<SyncMessage>();
 
         if (options.traceAs) {
-            return [Stream.fromQueue(queue).pipe(Stream.tap((msg) => Console.debug(
-                options.traceAs,
-                JSON.stringify(
-                    msg,
-                    (k, v) =>
-                        k === "changes" ||
-                        k === "encryptedChanges"
-                            ? v.slice(0, 20) + "..."
-                            : v,
-                    2,
+            return [
+                Stream.fromQueue(queue).pipe(
+                    Stream.tap((msg) =>
+                        Console.debug(
+                            options.traceAs,
+                            JSON.stringify(
+                                msg,
+                                (k, v) =>
+                                    k === "changes" || k === "encryptedChanges"
+                                        ? v.slice(0, 20) + "..."
+                                        : v,
+                                2,
+                            ),
+                        ),
+                    ),
                 ),
-            ))), queue];
+                queue,
+            ];
         } else {
             return [Stream.fromQueue(queue), queue];
         }

package/src/sync.ts

@@ -3,7 +3,7 @@ import { CoValueHeader, Transaction } from "./coValueCore.js";
 import { CoValueCore } from "./coValueCore.js";
 import { LocalNode, newLoadingState } from "./localNode.js";
 import { RawCoID, SessionID } from "./ids.js";
-import { Effect, Queue, Stream } from "effect";
+import { Data, Effect, Queue, Stream } from "effect";
 
 export type CoValueKnownState = {
     id: RawCoID;
@@ -56,12 +56,9 @@ export type DoneMessage = {
 
 export type PeerID = string;
 
-export class DisconnectedError extends Error {
-    readonly _tag = "DisconnectedError";
-    constructor(public message: string) {
-        super(message);
-    }
-}
+export class DisconnectedError extends Data.TaggedError("DisconnectedError")<{
+    message: string;
+}> {}
 
 export class PingTimeoutError extends Error {
     readonly _tag = "PingTimeoutError";

package/src/tests/account.test.ts

@@ -53,13 +53,15 @@ test("Can create account with one node, and then load it on another", async () =
     map.set("foo", "bar", "private");
     expect(map.get("foo")).toEqual("bar");
 
-    const [node1asPeer, node2asPeer] = await Effect.runPromise(connectedPeers("node1", "node2", {
-        trace: true,
-        peer1role: "server",
-        peer2role: "client",
-    }));
+    const [node1asPeer, node2asPeer] = await Effect.runPromise(
+        connectedPeers("node1", "node2", {
+            trace: true,
+            peer1role: "server",
+            peer2role: "client",
+        }),
+    );
 
-    console.log("After connected peers")
+    console.log("After connected peers");
 
     node.syncManager.addPeer(node2asPeer);