cojson 0.13.11 → 0.13.13

package/dist/storage/chunksAndKnownStates.js

@@ -1,98 +0,0 @@
-import { MAX_RECOMMENDED_TX_SIZE } from "../coValueCore.js";
-import { getPriorityFromHeader } from "../priority.js";
-export function contentSinceChunk(id, chunk, known) {
-    const newContentPieces = [];
-    newContentPieces.push({
-        id: id,
-        action: "content",
-        header: known?.header ? undefined : chunk.header,
-        new: {},
-        priority: getPriorityFromHeader(chunk.header),
-    });
-    for (const [sessionID, sessionsEntry] of Object.entries(chunk.sessionEntries)) {
-        for (const entry of sessionsEntry) {
-            const knownStart = known?.sessions[sessionID] || 0;
-            if (entry.after + entry.transactions.length <= knownStart) {
-                continue;
-            }
-            const actuallyNewTransactions = entry.transactions.slice(Math.max(0, knownStart - entry.after));
-            const newAfter = entry.after +
-                (actuallyNewTransactions.length - entry.transactions.length);
-            let newContentEntry = newContentPieces[0]?.new[sessionID];
-            if (!newContentEntry) {
-                newContentEntry = {
-                    after: newAfter,
-                    lastSignature: entry.lastSignature,
-                    newTransactions: actuallyNewTransactions,
-                };
-                newContentPieces[0].new[sessionID] = newContentEntry;
-            }
-            else {
-                newContentEntry.newTransactions.push(...actuallyNewTransactions);
-                newContentEntry.lastSignature = entry.lastSignature;
-            }
-        }
-    }
-    return newContentPieces;
-}
-export function chunkToKnownState(id, chunk) {
-    const ourKnown = {
-        id,
-        header: !!chunk.header,
-        sessions: {},
-    };
-    for (const [sessionID, sessionEntries] of Object.entries(chunk.sessionEntries)) {
-        for (const entry of sessionEntries) {
-            ourKnown.sessions[sessionID] =
-                entry.after + entry.transactions.length;
-        }
-    }
-    return ourKnown;
-}
-export function mergeChunks(chunkA, chunkB) {
-    const header = chunkA.header || chunkB.header;
-    const newSessions = { ...chunkA.sessionEntries };
-    for (const sessionID in chunkB.sessionEntries) {
-        // figure out if we can merge the chunks
-        const sessionEntriesA = chunkA.sessionEntries[sessionID];
-        const sessionEntriesB = chunkB.sessionEntries[sessionID];
-        if (!sessionEntriesA) {
-            newSessions[sessionID] = sessionEntriesB;
-            continue;
-        }
-        const lastEntryOfA = sessionEntriesA[sessionEntriesA.length - 1];
-        const firstEntryOfB = sessionEntriesB[0];
-        if (lastEntryOfA.after + lastEntryOfA.transactions.length ===
-            firstEntryOfB.after) {
-            const newEntries = [];
-            let bytesSinceLastSignature = 0;
-            for (const entry of sessionEntriesA.concat(sessionEntriesB)) {
-                const entryByteLength = entry.transactions.reduce((sum, tx) => sum +
-                    (tx.privacy === "private"
-                        ? tx.encryptedChanges.length
-                        : tx.changes.length), 0);
-                if (newEntries.length === 0 ||
-                    bytesSinceLastSignature + entryByteLength > MAX_RECOMMENDED_TX_SIZE) {
-                    newEntries.push({
-                        after: entry.after,
-                        lastSignature: entry.lastSignature,
-                        transactions: entry.transactions,
-                    });
-                    bytesSinceLastSignature = 0;
-                }
-                else {
-                    const lastNewEntry = newEntries[newEntries.length - 1];
-                    lastNewEntry.transactions.push(...entry.transactions);
-                    lastNewEntry.lastSignature = entry.lastSignature;
-                    bytesSinceLastSignature += entry.transactions.length;
-                }
-            }
-            newSessions[sessionID] = newEntries;
-        }
-        else {
-            return "nonContigous";
-        }
-    }
-    return { header, sessionEntries: newSessions };
-}
-//# sourceMappingURL=chunksAndKnownStates.js.map

package/dist/storage/chunksAndKnownStates.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"chunksAndKnownStates.js","sourceRoot":"","sources":["../../src/storage/chunksAndKnownStates.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAC;AAE5D,OAAO,EAAE,qBAAqB,EAAE,MAAM,gBAAgB,CAAC;AAIvD,MAAM,UAAU,iBAAiB,CAC/B,EAAW,EACX,KAAmB,EACnB,KAAyB;IAEzB,MAAM,gBAAgB,GAAwB,EAAE,CAAC;IAEjD,gBAAgB,CAAC,IAAI,CAAC;QACpB,EAAE,EAAE,EAAE;QACN,MAAM,EAAE,SAAS;QACjB,MAAM,EAAE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,MAAM;QAChD,GAAG,EAAE,EAAE;QACP,QAAQ,EAAE,qBAAqB,CAAC,KAAK,CAAC,MAAM,CAAC;KAC9C,CAAC,CAAC;IAEH,KAAK,MAAM,CAAC,SAAS,EAAE,aAAa,CAAC,IAAI,MAAM,CAAC,OAAO,CACrD,KAAK,CAAC,cAAc,CACrB,EAAE,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,aAAa,EAAE,CAAC;YAClC,MAAM,UAAU,GAAG,KAAK,EAAE,QAAQ,CAAC,SAAsB,CAAC,IAAI,CAAC,CAAC;YAEhE,IAAI,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,IAAI,UAAU,EAAE,CAAC;gBAC1D,SAAS;YACX,CAAC;YAED,MAAM,uBAAuB,GAAG,KAAK,CAAC,YAAY,CAAC,KAAK,CACtD,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,KAAK,CAAC,KAAK,CAAC,CACtC,CAAC;YAEF,MAAM,QAAQ,GACZ,KAAK,CAAC,KAAK;gBACX,CAAC,uBAAuB,CAAC,MAAM,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;YAE/D,IAAI,eAAe,GAAG,gBAAgB,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,SAAsB,CAAC,CAAC;YAEvE,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,eAAe,GAAG;oBAChB,KAAK,EAAE,QAAQ;oBACf,aAAa,EAAE,KAAK,CAAC,aAAa;oBAClC,eAAe,EAAE,uBAAuB;iBACzC,CAAC;gBACF,gBAAgB,CAAC,CAAC,CAAE,CAAC,GAAG,CAAC,SAAsB,CAAC,GAAG,eAAe,CAAC;YACrE,CAAC;iBAAM,CAAC;gBACN,eAAe,CAAC,eAAe,CAAC,IAAI,CAAC,GAAG,uBAAuB,CAAC,CAAC;gBACjE,eAAe,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;YACtD,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,EAAW,EAAE,KAAmB;IAChE,MAAM,QAAQ,GAAsB;QAClC,EAAE;QACF,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,MAAM;QACtB,QAAQ,EAAE,EAAE;KACb,CAAC;IAEF,KAAK,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,IAAI,MAAM,CAAC,OAAO,CACtD,KAAK,CAAC,cAAc,CACrB,EAAE,CAAC;QACF,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,QAAQ,CAAC,QAAQ,CAAC,SAAsB,CAAC;gBACvC,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,MAAoB,EACpB,MAAoB;IAEpB,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,MAAM,CAAC;IAE9C,MAAM,WAAW,GAAG,EAAE,GAAG,MAAM,CAAC,cAAc,EAAE,CAAC;IACjD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;QAC9C,wCAAwC;QACxC,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,CAAC,SAAS,CAAC,CAAC;QACzD,MAAM,eAAe,GAAG,MAAM,CAAC,cAAc,CAAC,SAAS,CAAE,CAAC;QAE1D,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,WAAW,CAAC,SAAS,CAAC,GAAG,eAAe,CAAC;YACzC,SAAS;QACX,CAAC;QAED,MAAM,YAAY,GAAG,eAAe,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;QAClE,MAAM,aAAa,GAAG,eAAe,CAAC,CAAC,CAAE,CAAC;QAE1C,IACE,YAAY,CAAC,KAAK,GAAG,YAAY,CAAC,YAAY,CAAC,MAAM;YACrD,aAAa,CAAC,KAAK,EACnB,CAAC;YACD,MAAM,UAAU,GAAG,EAAE,CAAC;YACtB,IAAI,uBAAuB,GAAG,CAAC,CAAC;YAChC,KAAK,MAAM,KAAK,IAAI,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,CAAC;gBAC5D,MAAM,eAAe,GAAG,KAAK,CAAC,YAAY,CAAC,MAAM,CAC/C,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE,CACV,GAAG;oBACH,CAAC,EAAE,CAAC,OAAO,KAAK,SAAS;wBACvB,CAAC,CAAC,EAAE,CAAC,gBAAgB,CAAC,MAAM;wBAC5B,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,EACxB,CAAC,CACF,CAAC;gBACF,IACE,UAAU,CAAC,MAAM,KAAK,CAAC;oBACvB,uBAAuB,GAAG,eAAe,GAAG,uBAAuB,EACnE,CAAC;oBACD,UAAU,CAAC,IAAI,CAAC;wBACd,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,aAAa,EAAE,KAAK,CAAC,aAAa;wBAClC,YAAY,EAAE,KAAK,CAAC,YAAY;qBACjC,CAAC,CAAC;oBACH,uBAAuB,GAAG,CAAC,CAAC;gBAC9B,CAAC;qBAAM,CAAC;oBACN,MAAM,YAAY,GAAG,UAAU,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAE,CAAC;oBACxD,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC,YAAY,CAAC,CAAC;oBACtD,YAAY,CAAC,aAAa,GAAG,KAAK,CAAC,aAAa,CAAC;oBAEjD,uBAAuB,IAAI,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC;gBACvD,CAAC;YACH,CAAC;YACD,WAAW,CAAC,SAAS,CAAC,GAAG,UAAU,CAAC;QACtC,CAAC;aAAM,CAAC;YACN,OAAO,cAAuB,CAAC;QACjC,CAAC;IACH,CAAC;IAED,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;AACjD,CAAC"}

package/dist/storage/index.d.ts

@@ -1,52 +0,0 @@
-import { CoValueHeader, Transaction } from "../coValueCore.js";
-import { Signature } from "../crypto/crypto.js";
-import { RawCoID } from "../ids.js";
-import { CoValueKnownState, IncomingSyncStream, NewContentMessage, OutgoingSyncQueue, Peer } from "../sync.js";
-import { BlockFilename, FileSystem } from "./FileSystem.js";
-export type { BlockFilename, WalFilename } from "./FileSystem.js";
-export type CoValueChunk = {
-    header?: CoValueHeader;
-    sessionEntries: {
-        [sessionID: string]: {
-            after: number;
-            lastSignature: Signature;
-            transactions: Transaction[];
-        }[];
-    };
-};
-export declare class LSMStorage<WH, RH, FS extends FileSystem<WH, RH>> {
-    fs: FS;
-    fromLocalNode: IncomingSyncStream;
-    toLocalNode: OutgoingSyncQueue;
-    currentWal: WH | undefined;
-    coValues: {
-        [id: RawCoID]: CoValueChunk | undefined;
-    };
-    fileCache: string[] | undefined;
-    headerCache: Map<`L${number}-${string}-${string}-H${number}.jsonl`, {
-        [id: `co_z${string}`]: {
-            start: number;
-            length: number;
-        };
-    }>;
-    blockFileHandles: Map<`L${number}-${string}-${string}-H${number}.jsonl`, Promise<{
-        handle: RH;
-        size: number;
-    }>>;
-    constructor(fs: FS, fromLocalNode: IncomingSyncStream, toLocalNode: OutgoingSyncQueue);
-    sendNewContent(id: RawCoID, known: CoValueKnownState | undefined, asDependencyOf: RawCoID | undefined): Promise<void>;
-    withWAL(handler: (wal: WH) => Promise<void>): Promise<void>;
-    handleNewContent(newContent: NewContentMessage): Promise<void>;
-    getBlockHandle(blockFile: BlockFilename, fs: FS): Promise<{
-        handle: RH;
-        size: number;
-    }>;
-    loadCoValue(id: RawCoID, fs: FS): Promise<CoValueChunk | undefined>;
-    compact(): Promise<void>;
-    static asPeer<WH, RH, FS extends FileSystem<WH, RH>>({ fs, trace, localNodeName, }: {
-        fs: FS;
-        trace?: boolean;
-        localNodeName?: string;
-    }): Peer;
-}
-//# sourceMappingURL=index.d.ts.map

package/dist/storage/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAChD,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,IAAI,EACL,MAAM,YAAY,CAAC;AACpB,OAAO,EACL,aAAa,EACb,UAAU,EAQX,MAAM,iBAAiB,CAAC;AAMzB,YAAY,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAIlE,MAAM,MAAM,YAAY,GAAG;IACzB,MAAM,CAAC,EAAE,aAAa,CAAC;IACvB,cAAc,EAAE;QACd,CAAC,SAAS,EAAE,MAAM,GAAG;YACnB,KAAK,EAAE,MAAM,CAAC;YACd,aAAa,EAAE,SAAS,CAAC;YACzB,YAAY,EAAE,WAAW,EAAE,CAAC;SAC7B,EAAE,CAAC;KACL,CAAC;CACH,CAAC;AAEF,qBAAa,UAAU,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,SAAS,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC;IAgBlD,EAAE,EAAE,EAAE;IACN,aAAa,EAAE,kBAAkB;IACjC,WAAW,EAAE,iBAAiB;IAjBvC,UAAU,EAAE,EAAE,GAAG,SAAS,CAAC;IAC3B,QAAQ,EAAE;QACR,CAAC,EAAE,EAAE,OAAO,GAAG,YAAY,GAAG,SAAS,CAAC;KACzC,CAAC;IACF,SAAS,EAAE,MAAM,EAAE,GAAG,SAAS,CAAC;IAChC,WAAW;;mBAEiB,MAAM;oBAAU,MAAM;;OAC9C;IACJ,gBAAgB;gBAEI,EAAE;cAAQ,MAAM;QAChC;gBAGK,EAAE,EAAE,EAAE,EACN,aAAa,EAAE,kBAAkB,EACjC,WAAW,EAAE,iBAAiB;IA6CjC,cAAc,CAClB,EAAE,EAAE,OAAO,EACX,KAAK,EAAE,iBAAiB,GAAG,SAAS,EACpC,cAAc,EAAE,OAAO,GAAG,SAAS;IAuF/B,OAAO,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE,KAAK,OAAO,CAAC,IAAI,CAAC;IAS3C,gBAAgB,CAAC,UAAU,EAAE,iBAAiB;IA6D9C,cAAc,CAClB,SAAS,EAAE,aAAa,EACxB,EAAE,EAAE,EAAE,GACL,OAAO,CAAC;QAAE,MAAM,EAAE,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;IAQlC,WAAW,CAAC,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC;IAuDnE,OAAO;IA4Kb,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,EAAE,EAAE,EAAE,SAAS,UAAU,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,EACnD,EAAE,EACF,KAAK,EACL,aAAuB,GACxB,EAAE;QACD,EAAE,EAAE,EAAE,CAAC;QACP,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,aAAa,CAAC,EAAE,MAAM,CAAC;KACxB,GAAG,IAAI;CAiBT"}

package/dist/storage/index.js

@@ -1,335 +0,0 @@
-import { logger } from "../logger.js";
-import { connectedPeers } from "../streamUtils.js";
-import { readChunk, readHeader, textDecoder, writeBlock, writeToWal, } from "./FileSystem.js";
-import { chunkToKnownState, contentSinceChunk, mergeChunks, } from "./chunksAndKnownStates.js";
-const MAX_N_LEVELS = 3;
-export class LSMStorage {
-    constructor(fs, fromLocalNode, toLocalNode) {
-        this.fs = fs;
-        this.fromLocalNode = fromLocalNode;
-        this.toLocalNode = toLocalNode;
-        this.headerCache = new Map();
-        this.blockFileHandles = new Map();
-        this.coValues = {};
-        this.currentWal = undefined;
-        let nMsg = 0;
-        const processMessages = async () => {
-            for await (const msg of fromLocalNode) {
-                try {
-                    if (msg === "Disconnected" || msg === "PingTimeout") {
-                        throw new Error("Unexpected Disconnected message");
-                    }
-                    if (msg.action === "done") {
-                        return;
-                    }
-                    if (msg.action === "content") {
-                        await this.handleNewContent(msg);
-                    }
-                    else if (msg.action === "load" || msg.action === "known") {
-                        await this.sendNewContent(msg.id, msg, undefined);
-                    }
-                }
-                catch (e) {
-                    logger.error(`Error reading from localNode, handling msg`, {
-                        msg,
-                        err: e,
-                    });
-                }
-                nMsg++;
-            }
-        };
-        processMessages().catch((e) => logger.error("Error in processMessages in storage", { err: e }));
-        setTimeout(() => this.compact().catch((e) => {
-            logger.error("Error while compacting", { err: e });
-        }), 20000);
-    }
-    async sendNewContent(id, known, asDependencyOf) {
-        let coValue = this.coValues[id];
-        if (!coValue) {
-            coValue = await this.loadCoValue(id, this.fs);
-        }
-        if (!coValue) {
-            this.toLocalNode
-                .push({
-                id: id,
-                action: "known",
-                header: false,
-                sessions: {},
-                asDependencyOf,
-            })
-                .catch((e) => logger.error("Error while pushing known", { err: e }));
-            return;
-        }
-        if (!known?.header && coValue.header?.ruleset.type === "ownedByGroup") {
-            await this.sendNewContent(coValue.header.ruleset.group, undefined, asDependencyOf || id);
-        }
-        else if (!known?.header && coValue.header?.ruleset.type === "group") {
-            const dependedOnAccountsAndGroups = new Set();
-            for (const session of Object.values(coValue.sessionEntries)) {
-                for (const entry of session) {
-                    for (const tx of entry.transactions) {
-                        if (tx.privacy === "trusting") {
-                            const parsedChanges = JSON.parse(tx.changes);
-                            for (const change of parsedChanges) {
-                                if (change.op === "set" && change.key.startsWith("co_")) {
-                                    dependedOnAccountsAndGroups.add(change.key);
-                                }
-                                if (change.op === "set" &&
-                                    change.key.startsWith("parent_co_")) {
-                                    dependedOnAccountsAndGroups.add(change.key.replace("parent_", ""));
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            for (const accountOrGroup of dependedOnAccountsAndGroups) {
-                await this.sendNewContent(accountOrGroup, undefined, asDependencyOf || id);
-            }
-        }
-        const newContentMessages = contentSinceChunk(id, coValue, known).map((message) => ({ ...message, asDependencyOf }));
-        const ourKnown = chunkToKnownState(id, coValue);
-        this.toLocalNode
-            .push({
-            action: "known",
-            ...ourKnown,
-            asDependencyOf,
-        })
-            .catch((e) => logger.error("Error while pushing known", { err: e }));
-        for (const message of newContentMessages) {
-            if (Object.keys(message.new).length === 0)
-                continue;
-            this.toLocalNode
-                .push(message)
-                .catch((e) => logger.error("Error while pushing new content", { err: e }));
-        }
-        this.coValues[id] = coValue;
-    }
-    async withWAL(handler) {
-        if (!this.currentWal) {
-            this.currentWal = await this.fs.createFile(`wal-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`);
-        }
-        await handler(this.currentWal);
-    }
-    async handleNewContent(newContent) {
-        const coValue = this.coValues[newContent.id];
-        const newContentAsChunk = {
-            header: newContent.header,
-            sessionEntries: Object.fromEntries(Object.entries(newContent.new).map(([sessionID, newInSession]) => [
-                sessionID,
-                [
-                    {
-                        after: newInSession.after,
-                        lastSignature: newInSession.lastSignature,
-                        transactions: newInSession.newTransactions,
-                    },
-                ],
-            ])),
-        };
-        if (!coValue) {
-            if (newContent.header) {
-                await this.withWAL((wal) => writeToWal(wal, this.fs, newContent.id, newContentAsChunk));
-                this.coValues[newContent.id] = newContentAsChunk;
-            }
-            else {
-                logger.warn("Incontiguous incoming update for " + newContent.id);
-                return;
-            }
-        }
-        else {
-            const merged = mergeChunks(coValue, newContentAsChunk);
-            if (merged === "nonContigous") {
-                console.warn("Non-contigous new content for " + newContent.id, Object.entries(coValue.sessionEntries).map(([session, entries]) => entries.map((entry) => ({
-                    session: session,
-                    after: entry.after,
-                    length: entry.transactions.length,
-                }))), Object.entries(newContentAsChunk.sessionEntries).map(([session, entries]) => entries.map((entry) => ({
-                    session: session,
-                    after: entry.after,
-                    length: entry.transactions.length,
-                }))));
-            }
-            else {
-                await this.withWAL((wal) => writeToWal(wal, this.fs, newContent.id, newContentAsChunk));
-                this.coValues[newContent.id] = merged;
-            }
-        }
-    }
-    async getBlockHandle(blockFile, fs) {
-        if (!this.blockFileHandles.has(blockFile)) {
-            this.blockFileHandles.set(blockFile, fs.openToRead(blockFile));
-        }
-        return this.blockFileHandles.get(blockFile);
-    }
-    async loadCoValue(id, fs) {
-        const files = this.fileCache || (await fs.listFiles());
-        this.fileCache = files;
-        const blockFiles = files.filter((name) => name.startsWith("L")).sort();
-        let result;
-        for (const blockFile of blockFiles) {
-            let cachedHeader = this.headerCache.get(blockFile);
-            const { handle, size } = await this.getBlockHandle(blockFile, fs);
-            if (!cachedHeader) {
-                cachedHeader = {};
-                const header = await readHeader(blockFile, handle, size, fs);
-                for (const entry of header) {
-                    cachedHeader[entry.id] = {
-                        start: entry.start,
-                        length: entry.length,
-                    };
-                }
-                this.headerCache.set(blockFile, cachedHeader);
-            }
-            const headerEntry = cachedHeader[id];
-            if (headerEntry) {
-                const nextChunk = await readChunk(handle, headerEntry, fs);
-                if (result) {
-                    const merged = mergeChunks(result, nextChunk);
-                    if (merged === "nonContigous") {
-                        console.warn("Non-contigous chunks while loading " + id, result, nextChunk);
-                    }
-                    else {
-                        result = merged;
-                    }
-                }
-                else {
-                    result = nextChunk;
-                }
-            }
-            // await fs.close(handle);
-        }
-        return result;
-    }
-    async compact() {
-        const fileNames = await this.fs.listFiles();
-        const walFiles = fileNames.filter((name) => name.startsWith("wal-"));
-        walFiles.sort();
-        const coValues = new Map();
-        if (walFiles.length === 0)
-            return;
-        const oldWal = this.currentWal;
-        this.currentWal = undefined;
-        if (oldWal) {
-            await this.fs.close(oldWal);
-        }
-        for (const fileName of walFiles) {
-            const { handle, size } = await this.fs.openToRead(fileName);
-            if (size === 0) {
-                await this.fs.close(handle);
-                continue;
-            }
-            const bytes = await this.fs.read(handle, 0, size);
-            const decoded = textDecoder.decode(bytes);
-            const lines = decoded.split("\n");
-            for (const line of lines) {
-                if (line.length === 0)
-                    continue;
-                const chunk = JSON.parse(line);
-                const existingChunk = coValues.get(chunk.id);
-                if (existingChunk) {
-                    const merged = mergeChunks(existingChunk, chunk);
-                    if (merged === "nonContigous") {
-                        console.log("Non-contigous chunks in " + chunk.id + ", " + fileName, existingChunk, chunk);
-                    }
-                    else {
-                        coValues.set(chunk.id, merged);
-                    }
-                }
-                else {
-                    coValues.set(chunk.id, chunk);
-                }
-            }
-            await this.fs.close(handle);
-        }
-        const highestBlockNumber = fileNames.reduce((acc, name) => {
-            if (name.startsWith("L" + MAX_N_LEVELS)) {
-                const num = parseInt(name.split("-")[1]);
-                if (num > acc) {
-                    return num;
-                }
-            }
-            return acc;
-        }, 0);
-        await writeBlock(coValues, MAX_N_LEVELS, highestBlockNumber + 1, this.fs);
-        for (const walFile of walFiles) {
-            await this.fs.removeFile(walFile);
-        }
-        this.fileCache = undefined;
-        const fileNames2 = await this.fs.listFiles();
-        const blockFiles = fileNames2.filter((name) => name.startsWith("L")).sort();
-        const blockFilesByLevelInOrder = {};
-        for (const blockFile of blockFiles) {
-            const level = parseInt(blockFile.split("-")[0].slice(1));
-            if (!blockFilesByLevelInOrder[level]) {
-                blockFilesByLevelInOrder[level] = [];
-            }
-            blockFilesByLevelInOrder[level].push(blockFile);
-        }
-        for (let level = MAX_N_LEVELS; level > 0; level--) {
-            const nBlocksDesired = Math.pow(2, level);
-            const blocksInLevel = blockFilesByLevelInOrder[level];
-            if (blocksInLevel && blocksInLevel.length > nBlocksDesired) {
-                const coValues = new Map();
-                for (const blockFile of blocksInLevel) {
-                    const { handle, size } = await this.getBlockHandle(blockFile, this.fs);
-                    if (size === 0) {
-                        continue;
-                    }
-                    const header = await readHeader(blockFile, handle, size, this.fs);
-                    for (const entry of header) {
-                        const chunk = await readChunk(handle, entry, this.fs);
-                        const existingChunk = coValues.get(entry.id);
-                        if (existingChunk) {
-                            const merged = mergeChunks(existingChunk, chunk);
-                            if (merged === "nonContigous") {
-                                console.log("Non-contigous chunks in " + entry.id + ", " + blockFile, existingChunk, chunk);
-                            }
-                            else {
-                                coValues.set(entry.id, merged);
-                            }
-                        }
-                        else {
-                            coValues.set(entry.id, chunk);
-                        }
-                    }
-                }
-                let levelBelow = blockFilesByLevelInOrder[level - 1];
-                if (!levelBelow) {
-                    levelBelow = [];
-                    blockFilesByLevelInOrder[level - 1] = levelBelow;
-                }
-                const highestBlockNumberInLevelBelow = levelBelow.reduce((acc, name) => {
-                    const num = parseInt(name.split("-")[1]);
-                    if (num > acc) {
-                        return num;
-                    }
-                    return acc;
-                }, 0);
-                const newBlockName = await writeBlock(coValues, level - 1, highestBlockNumberInLevelBelow + 1, this.fs);
-                levelBelow.push(newBlockName);
-                // delete blocks that went into this one
-                for (const blockFile of blocksInLevel) {
-                    const handle = await this.getBlockHandle(blockFile, this.fs);
-                    await this.fs.close(handle.handle);
-                    await this.fs.removeFile(blockFile);
-                    this.blockFileHandles.delete(blockFile);
-                }
-            }
-        }
-        setTimeout(() => this.compact().catch((e) => {
-            logger.error("Error while compacting", { err: e });
-        }), 5000);
-    }
-    static asPeer({ fs, trace, localNodeName = "local", }) {
-        const [localNodeAsPeer, storageAsPeer] = connectedPeers(localNodeName, "storage", {
-            peer1role: "client",
-            peer2role: "storage",
-            trace,
-            crashOnClose: true,
-        });
-        new LSMStorage(fs, localNodeAsPeer.incoming, localNodeAsPeer.outgoing);
-        // return { ...storageAsPeer, priority: 200 };
-        return storageAsPeer;
-    }
-}
-//# sourceMappingURL=index.js.map

package/dist/storage/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/storage/index.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AACtC,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAQnD,OAAO,EAKL,SAAS,EACT,UAAU,EACV,WAAW,EACX,UAAU,EACV,UAAU,GACX,MAAM,iBAAiB,CAAC;AACzB,OAAO,EACL,iBAAiB,EACjB,iBAAiB,EACjB,WAAW,GACZ,MAAM,2BAA2B,CAAC;AAGnC,MAAM,YAAY,GAAG,CAAC,CAAC;AAavB,MAAM,OAAO,UAAU;IAerB,YACS,EAAM,EACN,aAAiC,EACjC,WAA8B;QAF9B,OAAE,GAAF,EAAE,CAAI;QACN,kBAAa,GAAb,aAAa,CAAoB;QACjC,gBAAW,GAAX,WAAW,CAAmB;QAZvC,gBAAW,GAAG,IAAI,GAAG,EAGlB,CAAC;QACJ,qBAAgB,GAAG,IAAI,GAAG,EAGvB,CAAC;QAOF,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC;QACnB,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,IAAI,IAAI,GAAG,CAAC,CAAC;QAEb,MAAM,eAAe,GAAG,KAAK,IAAI,EAAE;YACjC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;gBACtC,IAAI,CAAC;oBACH,IAAI,GAAG,KAAK,cAAc,IAAI,GAAG,KAAK,aAAa,EAAE,CAAC;wBACpD,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;oBACrD,CAAC;oBACD,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;wBAC1B,OAAO;oBACT,CAAC;oBAED,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;wBAC7B,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;oBACnC,CAAC;yBAAM,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,MAAM,KAAK,OAAO,EAAE,CAAC;wBAC3D,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,EAAE,SAAS,CAAC,CAAC;oBACpD,CAAC;gBACH,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,MAAM,CAAC,KAAK,CAAC,4CAA4C,EAAE;wBACzD,GAAG;wBACH,GAAG,EAAE,CAAC;qBACP,CAAC,CAAC;gBACL,CAAC;gBACD,IAAI,EAAE,CAAC;YACT,CAAC;QACH,CAAC,CAAC;QAEF,eAAe,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5B,MAAM,CAAC,KAAK,CAAC,qCAAqC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAChE,CAAC;QAEF,UAAU,CACR,GAAG,EAAE,CACH,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC,CAAC,EACJ,KAAK,CACN,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,EAAW,EACX,KAAoC,EACpC,cAAmC;QAEnC,IAAI,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAEhC,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,CAAC,WAAW;iBACb,IAAI,CAAC;gBACJ,EAAE,EAAE,EAAE;gBACN,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,KAAK;gBACb,QAAQ,EAAE,EAAE;gBACZ,cAAc;aACf,CAAC;iBACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;YAEvE,OAAO;QACT,CAAC;QAED,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,cAAc,EAAE,CAAC;YACtE,MAAM,IAAI,CAAC,cAAc,CACvB,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAC5B,SAAS,EACT,cAAc,IAAI,EAAE,CACrB,CAAC;QACJ,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,MAAM,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YACtE,MAAM,2BAA2B,GAAG,IAAI,GAAG,EAAE,CAAC;YAC9C,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;gBAC5D,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;oBAC5B,KAAK,MAAM,EAAE,IAAI,KAAK,CAAC,YAAY,EAAE,CAAC;wBACpC,IAAI,EAAE,CAAC,OAAO,KAAK,UAAU,EAAE,CAAC;4BAC9B,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;4BAC7C,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;gCACnC,IAAI,MAAM,CAAC,EAAE,KAAK,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;oCACxD,2BAA2B,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;gCAC9C,CAAC;gCACD,IACE,MAAM,CAAC,EAAE,KAAK,KAAK;oCACnB,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC,YAAY,CAAC,EACnC,CAAC;oCACD,2BAA2B,CAAC,GAAG,CAC7B,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,SAAS,EAAE,EAAE,CAAC,CAClC,CAAC;gCACJ,CAAC;4BACH,CAAC;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YACD,KAAK,MAAM,cAAc,IAAI,2BAA2B,EAAE,CAAC;gBACzD,MAAM,IAAI,CAAC,cAAc,CACvB,cAAkC,EAClC,SAAS,EACT,cAAc,IAAI,EAAE,CACrB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,EAAE,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC,GAAG,CAClE,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,OAAO,EAAE,cAAc,EAAE,CAAC,CAC9C,CAAC;QAEF,MAAM,QAAQ,GAAsB,iBAAiB,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAEnE,IAAI,CAAC,WAAW;aACb,IAAI,CAAC;YACJ,MAAM,EAAE,OAAO;YACf,GAAG,QAAQ;YACX,cAAc;SACf,CAAC;aACD,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;QAEvE,KAAK,MAAM,OAAO,IAAI,kBAAkB,EAAE,CAAC;YACzC,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YACpD,IAAI,CAAC,WAAW;iBACb,IAAI,CAAC,OAAO,CAAC;iBACb,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CACX,MAAM,CAAC,KAAK,CAAC,iCAAiC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAC5D,CAAC;QACN,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC;IAC9B,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,OAAmC;QAC/C,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,IAAI,CAAC,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CACxC,OAAO,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,CACjE,CAAC;QACJ,CAAC;QACD,MAAM,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACjC,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAA6B;QAClD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;QAE7C,MAAM,iBAAiB,GAAiB;YACtC,MAAM,EAAE,UAAU,CAAC,MAAM;YACzB,cAAc,EAAE,MAAM,CAAC,WAAW,CAChC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,YAAY,CAAC,EAAE,EAAE,CAAC;gBAChE,SAAS;gBACT;oBACE;wBACE,KAAK,EAAE,YAAY,CAAC,KAAK;wBACzB,aAAa,EAAE,YAAY,CAAC,aAAa;wBACzC,YAAY,EAAE,YAAY,CAAC,eAAe;qBAC3C;iBACF;aACF,CAAC,CACH;SACF,CAAC;QAEF,IAAI,CAAC,OAAO,EAAE,CAAC;YACb,IAAI,UAAU,CAAC,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAC3D,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,iBAAiB,CAAC;YACnD,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,mCAAmC,GAAG,UAAU,CAAC,EAAE,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,WAAW,CAAC,OAAO,EAAE,iBAAiB,CAAC,CAAC;YACvD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CACV,gCAAgC,GAAG,UAAU,CAAC,EAAE,EAChD,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CAChE,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACtB,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;iBAClC,CAAC,CAAC,CACJ,EACD,MAAM,CAAC,OAAO,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,GAAG,CAClD,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,EAAE,EAAE,CACrB,OAAO,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;oBACtB,OAAO,EAAE,OAAO;oBAChB,KAAK,EAAE,KAAK,CAAC,KAAK;oBAClB,MAAM,EAAE,KAAK,CAAC,YAAY,CAAC,MAAM;iBAClC,CAAC,CAAC,CACN,CACF,CAAC;YACJ,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CACzB,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,EAAE,UAAU,CAAC,EAAE,EAAE,iBAAiB,CAAC,CAC3D,CAAC;gBAEF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC;YACxC,CAAC;QACH,CAAC;IACH,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,SAAwB,EACxB,EAAM;QAEN,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,EAAE,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,SAAS,CAAE,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,EAAW,EAAE,EAAM;QACnC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,IAAI,CAAC,MAAM,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;QACvB,MAAM,UAAU,GACd,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAC5C,CAAC,IAAI,EAAE,CAAC;QAET,IAAI,MAAM,CAAC;QAEX,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,IAAI,YAAY,GAEA,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAEhD,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC;YAElE,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,YAAY,GAAG,EAAE,CAAC;gBAClB,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC;gBAC7D,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBAC3B,YAAY,CAAC,KAAK,CAAC,EAAE,CAAC,GAAG;wBACvB,KAAK,EAAE,KAAK,CAAC,KAAK;wBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;qBACrB,CAAC;gBACJ,CAAC;gBAED,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAS,EAAE,YAAY,CAAC,CAAC;YAChD,CAAC;YACD,MAAM,WAAW,GAAG,YAAY,CAAC,EAAE,CAAC,CAAC;YAErC,IAAI,WAAW,EAAE,CAAC;gBAChB,MAAM,SAAS,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;gBAC3D,IAAI,MAAM,EAAE,CAAC;oBACX,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;oBAE9C,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC9B,OAAO,CAAC,IAAI,CACV,qCAAqC,GAAG,EAAE,EAC1C,MAAM,EACN,SAAS,CACV,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,MAAM,GAAG,MAAM,CAAC;oBAClB,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,MAAM,GAAG,SAAS,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,0BAA0B;QAC5B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;QAE5C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CACzC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CACP,CAAC;QACnB,QAAQ,CAAC,IAAI,EAAE,CAAC;QAEhB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;QAElD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAElC,MAAM,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC;QAC/B,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC;QAE5B,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QAED,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GACpB,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YACrC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBAC5B,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;YAElD,MAAM,OAAO,GAAG,WAAW,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC1C,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;oBAAE,SAAS;gBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAa,CAAC;gBAE3C,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBAE7C,IAAI,aAAa,EAAE,CAAC;oBAClB,MAAM,MAAM,GAAG,WAAW,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;oBACjD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;wBAC9B,OAAO,CAAC,GAAG,CACT,0BAA0B,GAAG,KAAK,CAAC,EAAE,GAAG,IAAI,GAAG,QAAQ,EACvD,aAAa,EACb,KAAK,CACN,CAAC;oBACJ,CAAC;yBAAM,CAAC;wBACN,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;oBACjC,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;gBAChC,CAAC;YACH,CAAC;YAED,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,kBAAkB,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;YACxD,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,CAAC;gBACxC,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;gBAC1C,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;oBACd,OAAO,GAAG,CAAC;gBACb,CAAC;YACH,CAAC;YACD,OAAO,GAAG,CAAC;QACb,CAAC,EAAE,CAAC,CAAC,CAAC;QAEN,MAAM,UAAU,CAAC,QAAQ,EAAE,YAAY,EAAE,kBAAkB,GAAG,CAAC,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;QAE1E,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QACpC,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAE3B,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;QAE7C,MAAM,UAAU,GACd,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CACjD,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,wBAAwB,GAE1B,EAAE,CAAC;QAEP,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,QAAQ,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;YAC1D,IAAI,CAAC,wBAAwB,CAAC,KAAK,CAAC,EAAE,CAAC;gBACrC,wBAAwB,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YACvC,CAAC;YACD,wBAAwB,CAAC,KAAK,CAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACnD,CAAC;QAED,KAAK,IAAI,KAAK,GAAG,YAAY,EAAE,KAAK,GAAG,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC;YAClD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YAC1C,MAAM,aAAa,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;YAEtD,IAAI,aAAa,IAAI,aAAa,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;gBAC3D,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAyB,CAAC;gBAElD,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;oBACtC,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,GACpB,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAEhD,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;wBACf,SAAS;oBACX,CAAC;oBACD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAClE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;wBAC3B,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;wBAEtD,MAAM,aAAa,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;wBAE7C,IAAI,aAAa,EAAE,CAAC;4BAClB,MAAM,MAAM,GAAG,WAAW,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC;4BACjD,IAAI,MAAM,KAAK,cAAc,EAAE,CAAC;gCAC9B,OAAO,CAAC,GAAG,CACT,0BAA0B,GAAG,KAAK,CAAC,EAAE,GAAG,IAAI,GAAG,SAAS,EACxD,aAAa,EACb,KAAK,CACN,CAAC;4BACJ,CAAC;iCAAM,CAAC;gCACN,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,MAAM,CAAC,CAAC;4BACjC,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;wBAChC,CAAC;oBACH,CAAC;gBACH,CAAC;gBAED,IAAI,UAAU,GAAG,wBAAwB,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;gBACrD,IAAI,CAAC,UAAU,EAAE,CAAC;oBAChB,UAAU,GAAG,EAAE,CAAC;oBAChB,wBAAwB,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,UAAU,CAAC;gBACnD,CAAC;gBAED,MAAM,8BAA8B,GAAG,UAAU,CAAC,MAAM,CACtD,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;oBACZ,MAAM,GAAG,GAAG,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC,CAAC;oBAC1C,IAAI,GAAG,GAAG,GAAG,EAAE,CAAC;wBACd,OAAO,GAAG,CAAC;oBACb,CAAC;oBACD,OAAO,GAAG,CAAC;gBACb,CAAC,EACD,CAAC,CACF,CAAC;gBAEF,MAAM,YAAY,GAAG,MAAM,UAAU,CACnC,QAAQ,EACR,KAAK,GAAG,CAAC,EACT,8BAA8B,GAAG,CAAC,EAClC,IAAI,CAAC,EAAE,CACR,CAAC;gBACF,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAE9B,wCAAwC;gBACxC,KAAK,MAAM,SAAS,IAAI,aAAa,EAAE,CAAC;oBACtC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,CAAC,EAAE,CAAC,CAAC;oBAC7D,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;oBACnC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;oBACpC,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;gBAC1C,CAAC;YACH,CAAC;QACH,CAAC;QAED,UAAU,CACR,GAAG,EAAE,CACH,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE;YACzB,MAAM,CAAC,KAAK,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC,CAAC,EACJ,IAAI,CACL,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,MAAM,CAAwC,EACnD,EAAE,EACF,KAAK,EACL,aAAa,GAAG,OAAO,GAKxB;QACC,MAAM,CAAC,eAAe,EAAE,aAAa,CAAC,GAAG,cAAc,CACrD,aAAa,EACb,SAAS,EACT;YACE,SAAS,EAAE,QAAQ;YACnB,SAAS,EAAE,SAAS;YACpB,KAAK;YACL,YAAY,EAAE,IAAI;SACnB,CACF,CAAC;QAEF,IAAI,UAAU,CAAC,EAAE,EAAE,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,QAAQ,CAAC,CAAC;QAEvE,8CAA8C;QAC9C,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}

package/src/storage/FileSystem.ts

@@ -1,113 +0,0 @@
-import { CryptoProvider, StreamingHash } from "../crypto/crypto.js";
-import { RawCoID } from "../ids.js";
-import { CoValueChunk } from "./index.js";
-
-export type BlockFilename = `L${number}-${string}-${string}-H${number}.jsonl`;
-
-export type BlockHeader = { id: RawCoID; start: number; length: number }[];
-
-export type WalEntry = { id: RawCoID } & CoValueChunk;
-
-export type WalFilename = `wal-${number}.jsonl`;
-
-export interface FileSystem<WriteHandle, ReadHandle> {
-  crypto: CryptoProvider;
-  createFile(filename: string): Promise<WriteHandle>;
-  append(handle: WriteHandle, data: Uint8Array): Promise<void>;
-  close(handle: ReadHandle | WriteHandle): Promise<void>;
-  closeAndRename(handle: WriteHandle, filename: BlockFilename): Promise<void>;
-  openToRead(filename: string): Promise<{ handle: ReadHandle; size: number }>;
-  read(handle: ReadHandle, offset: number, length: number): Promise<Uint8Array>;
-  listFiles(): Promise<string[]>;
-  removeFile(filename: BlockFilename | WalFilename): Promise<void>;
-}
-
-export const textEncoder = new TextEncoder();
-export const textDecoder = new TextDecoder();
-
-export async function readChunk<RH, FS extends FileSystem<unknown, RH>>(
-  handle: RH,
-  header: { start: number; length: number },
-  fs: FS,
-): Promise<CoValueChunk> {
-  const chunkBytes = await fs.read(handle, header.start, header.length);
-
-  const chunk = JSON.parse(textDecoder.decode(chunkBytes));
-  return chunk;
-}
-
-export async function readHeader<RH, FS extends FileSystem<unknown, RH>>(
-  filename: string,
-  handle: RH,
-  size: number,
-  fs: FS,
-): Promise<BlockHeader> {
-  const headerLength = Number(filename.match(/-H(\d+)\.jsonl$/)![1]!);
-
-  const headerBytes = await fs.read(handle, size - headerLength, headerLength);
-
-  const header = JSON.parse(textDecoder.decode(headerBytes));
-  return header;
-}
-
-export async function writeBlock<WH, RH, FS extends FileSystem<WH, RH>>(
-  chunks: Map<RawCoID, CoValueChunk>,
-  level: number,
-  blockNumber: number,
-  fs: FS,
-): Promise<BlockFilename> {
-  if (chunks.size === 0) {
-    throw new Error("No chunks to write");
-  }
-
-  const blockHeader: BlockHeader = [];
-
-  let offset = 0;
-
-  const file = await fs.createFile(
-    "wipBlock" + Math.random().toString(36).substring(7) + ".tmp.jsonl",
-  );
-  const hash = new StreamingHash(fs.crypto);
-
-  const chunksSortedById = Array.from(chunks).sort(([id1], [id2]) =>
-    id1.localeCompare(id2),
-  );
-
-  for (const [id, chunk] of chunksSortedById) {
-    const encodedBytes = hash.update(chunk);
-    const encodedBytesWithNewline = new Uint8Array(encodedBytes.length + 1);
-    encodedBytesWithNewline.set(encodedBytes);
-    encodedBytesWithNewline[encodedBytes.length] = 10;
-    await fs.append(file, encodedBytesWithNewline);
-    const length = encodedBytesWithNewline.length;
-    blockHeader.push({ id, start: offset, length });
-    offset += length;
-  }
-
-  const headerBytes = textEncoder.encode(JSON.stringify(blockHeader));
-  await fs.append(file, headerBytes);
-
-  const filename: BlockFilename = `L${level}-${(blockNumber + "").padStart(
-    3,
-    "0",
-  )}-${hash.digest().replace("hash_", "").slice(0, 15)}-H${
-    headerBytes.length
-  }.jsonl`;
-  await fs.closeAndRename(file, filename);
-
-  return filename;
-}
-
-export async function writeToWal<WH, RH, FS extends FileSystem<WH, RH>>(
-  handle: WH,
-  fs: FS,
-  id: RawCoID,
-  chunk: CoValueChunk,
-) {
-  const walEntry: WalEntry = {
-    id,
-    ...chunk,
-  };
-  const bytes = textEncoder.encode(JSON.stringify(walEntry) + "\n");
-  return fs.append(handle, bytes);
-}