@statewalker/cellb 0.0.0-stage → 0.1.1

package/dist/ltx/ltx-pages.js

@@ -0,0 +1,315 @@
+import { Crc64 } from "./crc64.js";
+import { CHECKSUM_FLAG, lockPgno } from "./ltx.js";
+import { lz4Compress, lz4Decompress, lz4FrameDecompress } from "./lz4.js";
+const HEADER_SIZE = 100;
+const PAGE_HEADER_FLAG_SIZE = 1;
+async function readRange(files, path, start, length) {
+    const chunks = [];
+    for await (const c of files.read(path, { start, length }))
+        chunks.push(c);
+    const got = chunks.reduce((n, c) => n + c.length, 0);
+    // A short read is a file that changed or went away under its reader: say which, rather than
+    // decode past its end ("Offset is outside the bounds of the DataView").
+    if (got !== length) {
+        const st = await files.stats(path).catch(() => undefined);
+        throw new LtxShortReadError(`ltx: short read of ${path}: ${got} of ${length} bytes at ${start} (file ${st?.kind === "file" ? `${st.size} bytes` : "missing"})`);
+    }
+    if (chunks.length === 1)
+        return chunks[0];
+    const out = new Uint8Array(got);
+    let o = 0;
+    for (const c of chunks)
+        out.set(c, o), (o += c.length);
+    return out;
+}
+/** A range read that got fewer bytes than asked: the file changed or was removed meanwhile. */
+export class LtxShortReadError extends Error {
+    name = "LtxShortReadError";
+}
+/** Opens an LTX file for page reads: reads its header and its page index only. */
+export async function openLtxPages(files, path, 
+/** The header and index of a file this process just wrote: nothing is read back. */
+known) {
+    if (known)
+        return pagesOf(files, path, known.header, known.index);
+    const size = (await files.stats(path))?.kind === "file"
+        ? (await files.stats(path)).size
+        : 0;
+    if (size < HEADER_SIZE + 6 + 24)
+        throw new Error(`ltx: ${path} is too small`);
+    const head = await readRange(files, path, 0, HEADER_SIZE);
+    const hv = new DataView(head.buffer, head.byteOffset, head.byteLength);
+    if (new TextDecoder().decode(head.subarray(0, 4)) !== "LTX1")
+        throw new Error(`ltx: ${path}: bad magic`);
+    const header = {
+        flags: hv.getUint32(4),
+        pageSize: hv.getUint32(8),
+        commit: hv.getUint32(12),
+        minTXID: hv.getBigUint64(16),
+        maxTXID: hv.getBigUint64(24),
+        timestamp: Number(hv.getBigInt64(32)),
+        preApplyChecksum: hv.getBigUint64(40),
+    };
+    // Trailer: … index | u64 index length | u64 post-apply checksum | u64 file checksum.
+    const tail = await readRange(files, path, size - 24, 24);
+    const idxLen = Number(new DataView(tail.buffer, tail.byteOffset, 24).getBigUint64(0));
+    const idx = await readRange(files, path, size - 24 - idxLen, idxLen);
+    const index = new Map();
+    let o = 0;
+    const varint = () => {
+        let x = 0;
+        let shift = 0;
+        let b;
+        do {
+            b = idx[o++];
+            x += (b & 0x7f) * 2 ** shift;
+            shift += 7;
+        } while (b & 0x80);
+        return x;
+    };
+    for (;;) {
+        const pgno = varint();
+        if (pgno === 0)
+            break;
+        const offset = varint();
+        index.set(pgno, [offset, varint()]);
+    }
+    return pagesOf(files, path, header, index);
+}
+function pagesOf(files, path, header, index) {
+    const decodePage = (b) => {
+        const v = new DataView(b.buffer, b.byteOffset, b.byteLength);
+        if (v.getUint16(4) & PAGE_HEADER_FLAG_SIZE)
+            return lz4Decompress(b.subarray(10, 10 + v.getUint32(6)), header.pageSize);
+        return lz4FrameDecompress(b.subarray(6), header.pageSize);
+    };
+    return {
+        path,
+        header,
+        index,
+        async readPage(pgno) {
+            const at = index.get(pgno);
+            if (!at)
+                return undefined;
+            return decodePage(await readRange(files, path, at[0], at[1]));
+        },
+        async readPages(pgnos) {
+            const out = new Map();
+            const wanted = pgnos.filter((p) => index.has(p));
+            if (!wanted.length)
+                return out;
+            const spans = wanted.map((p) => index.get(p));
+            const start = Math.min(...spans.map((s) => s[0]));
+            const end = Math.max(...spans.map((s) => s[0] + s[1]));
+            const b = await readRange(files, path, start, end - start);
+            for (const [i, p] of wanted.entries())
+                out.set(p, decodePage(b.subarray(spans[i][0] - start, spans[i][0] - start + spans[i][1])));
+            return out;
+        },
+    };
+}
+/**
+ * A cell's pages over its LTX files: the newest snapshot (min TXID 1) and the files after it,
+ * in TXID order. Only headers and page indexes are read.
+ */
+export async function openPagedCell(files, paths) {
+    const opened = await Promise.all(paths.map((p) => openLtxPages(files, p)));
+    opened.sort((a, b) => a.header.maxTXID < b.header.maxTXID ? -1 : a.header.maxTXID > b.header.maxTXID ? 1 : 0);
+    let snapshotAt = -1;
+    opened.forEach((f, i) => {
+        if (f.header.minTXID === 1n)
+            snapshotAt = i;
+    });
+    if (snapshotAt < 0)
+        throw new Error("ltx: no snapshot among the cell's files");
+    const chain = opened.slice(snapshotAt);
+    for (let i = 1; i < chain.length; i++)
+        if (chain[i].header.minTXID !== chain[i - 1].header.maxTXID + 1n)
+            throw new Error(`ltx: gap before ${chain[i].path}`);
+    let newest = chain[chain.length - 1].header;
+    const owner = new Map();
+    for (const f of chain)
+        for (const pgno of f.index.keys())
+            owner.set(pgno, f);
+    const cell = {
+        pageSize: newest.pageSize,
+        pageCount: newest.commit,
+        maxTXID: newest.maxTXID,
+        files: chain,
+        source: (pgno) => owner.get(pgno)?.path,
+        async readPage(pgno) {
+            const page = pgno <= cell.pageCount ? await owner.get(pgno)?.readPage(pgno) : undefined;
+            return page ?? new Uint8Array(cell.pageSize);
+        },
+        async readRun(from, count) {
+            const byFile = new Map();
+            const out = new Map();
+            for (let p = from; p < from + count && p <= cell.pageCount; p++) {
+                const f = owner.get(p);
+                if (!f)
+                    out.set(p, new Uint8Array(cell.pageSize));
+                else
+                    byFile.set(f, [...(byFile.get(f) ?? []), p]);
+            }
+            for (const [f, pgnos] of byFile)
+                for (const [p, d] of await f.readPages(pgnos))
+                    out.set(p, d);
+            return out;
+        },
+        async append(path, known) {
+            const f = await openLtxPages(files, path, known);
+            if (f.header.minTXID !== cell.maxTXID + 1n)
+                throw new Error(`ltx: ${path} does not follow TXID ${cell.maxTXID}`);
+            chain.push(f);
+            for (const pgno of f.index.keys())
+                owner.set(pgno, f);
+            newest = f.header;
+            cell.pageCount = newest.commit;
+            cell.maxTXID = newest.maxTXID;
+            for (const [pgno] of owner)
+                if (pgno > cell.pageCount)
+                    owner.delete(pgno);
+        },
+    };
+    return cell;
+}
+/**
+ * An LTX file produced page by page, as celld writes it (block format: size-prefixed LZ4 page
+ * blocks, page index, trailer, CRC-64/ISO over the uncompressed pages): the bytes come out as an
+ * async stream while `pages` is consumed, so a writer holds one page at a time plus the index
+ * (a few bytes per page), never the database.
+ */
+export async function* encodeLtxStream(header, pages, postApplyChecksum = 0n, 
+/** Filled while encoding: the file's page index (what `openLtxPages` would read back). */
+written) {
+    // Coalesced into 256 KiB chunks: a writer pays per call (an OPFS SyncAccessHandle write, a
+    // stream write), and a page makes three small pieces.
+    yield* coalesce(encodePieces(header, pages, postApplyChecksum, written), 256 * 1024);
+}
+async function* coalesce(src, size) {
+    let buf = new Uint8Array(size);
+    let n = 0;
+    for await (const piece of src) {
+        if (piece.length >= size) {
+            if (n)
+                yield buf.slice(0, n), (n = 0);
+            yield piece;
+            continue;
+        }
+        if (n + piece.length > size) {
+            yield buf.slice(0, n);
+            buf = new Uint8Array(size);
+            n = 0;
+        }
+        buf.set(piece, n);
+        n += piece.length;
+    }
+    if (n)
+        yield buf.slice(0, n);
+}
+async function* encodePieces(header, pages, postApplyChecksum, written) {
+    const crc = new Crc64();
+    const head = new Uint8Array(HEADER_SIZE);
+    const v = new DataView(head.buffer);
+    head.set(new TextEncoder().encode("LTX1"), 0);
+    const flags = header.flags ?? (header.preApplyChecksum === 0n && postApplyChecksum === 0n ? 2 : 0);
+    v.setUint32(4, flags);
+    v.setUint32(8, header.pageSize);
+    v.setUint32(12, header.commit);
+    v.setBigUint64(16, header.minTXID);
+    v.setBigUint64(24, header.maxTXID);
+    v.setBigInt64(32, BigInt(header.timestamp));
+    v.setBigUint64(40, header.preApplyChecksum);
+    v.setBigUint64(72, header.nodeID ?? 0n);
+    crc.update(head);
+    yield head;
+    const index = [];
+    const uvarint = (n) => {
+        while (n >= 0x80)
+            index.push((n & 0x7f) | 0x80), (n = Math.floor(n / 128));
+        index.push(n);
+    };
+    let offset = HEADER_SIZE;
+    const lock = lockPgno(header.pageSize);
+    let last = 0;
+    for await (const [pgno, data] of pages) {
+        if (pgno === lock)
+            continue;
+        if (pgno <= last)
+            throw new Error("ltx: pages must come in ascending order");
+        last = pgno;
+        const compressed = lz4Compress(data);
+        const ph = new Uint8Array(10);
+        const pv = new DataView(ph.buffer);
+        pv.setUint32(0, pgno);
+        pv.setUint16(4, PAGE_HEADER_FLAG_SIZE);
+        pv.setUint32(6, compressed.length);
+        crc.update(ph).update(data); // the checksum covers uncompressed pages (encodeLtx's `feed`)
+        uvarint(pgno), uvarint(offset), uvarint(10 + compressed.length);
+        written?.index.set(pgno, [offset, 10 + compressed.length]);
+        offset += 10 + compressed.length;
+        yield ph;
+        yield compressed;
+    }
+    const end = new Uint8Array(6);
+    uvarint(0);
+    const idx = Uint8Array.from(index);
+    const tail = new Uint8Array(16);
+    const tv = new DataView(tail.buffer);
+    tv.setBigUint64(0, BigInt(idx.length));
+    tv.setBigUint64(8, postApplyChecksum);
+    crc.update(end).update(idx).update(tail);
+    const sum = new Uint8Array(8);
+    new DataView(sum.buffer).setBigUint64(0, CHECKSUM_FLAG | crc.digest());
+    yield end;
+    yield idx;
+    yield tail;
+    yield sum;
+}
+/**
+ * Folds a cell's files into one snapshot (TXID 1..maxTXID) written to `files` at `path`, page by
+ * page: each page is read by range from the file that holds it and streamed into the new file,
+ * so memory stays at one page (plus the index), whatever the database size. The caller removes
+ * the files it replaces once this resolves.
+ */
+export async function compactPagedCellTo(cell, files, path, timestamp = Date.now()) {
+    // In runs of 256 pages: one range read per file a run touches (not one read per page).
+    async function* pages() {
+        for (let from = 1; from <= cell.pageCount; from += 256) {
+            const run = await cell.readRun(from, 256);
+            for (let pgno = from; pgno < from + 256 && pgno <= cell.pageCount; pgno++)
+                yield [pgno, run.get(pgno) ?? new Uint8Array(cell.pageSize)];
+        }
+    }
+    await files.write(path, encodeLtxStream({
+        pageSize: cell.pageSize,
+        commit: cell.pageCount,
+        minTXID: 1n,
+        maxTXID: cell.maxTXID,
+        timestamp,
+        preApplyChecksum: 0n,
+    }, pages()));
+}
+/** The compacted snapshot as bytes (small cells, tests); compactPagedCellTo streams it. */
+export async function compactPagedCell(cell, timestamp = Date.now()) {
+    const chunks = [];
+    async function* pages() {
+        for (let pgno = 1; pgno <= cell.pageCount; pgno++)
+            yield [pgno, await cell.readPage(pgno)];
+    }
+    for await (const c of encodeLtxStream({
+        pageSize: cell.pageSize,
+        commit: cell.pageCount,
+        minTXID: 1n,
+        maxTXID: cell.maxTXID,
+        timestamp,
+        preApplyChecksum: 0n,
+    }, pages()))
+        chunks.push(c);
+    const out = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0));
+    let o = 0;
+    for (const c of chunks)
+        out.set(c, o), (o += c.length);
+    return out;
+}
+//# sourceMappingURL=ltx-pages.js.map

package/dist/ltx/ltx-pages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ltx-pages.js","sourceRoot":"","sources":["../../src/ltx/ltx-pages.ts"],"names":[],"mappings":"AAYA,OAAO,EAAE,KAAK,EAAE,MAAM,YAAY,CAAC;AACnC,OAAO,EAAE,aAAa,EAAkB,QAAQ,EAAE,MAAM,UAAU,CAAC;AACnE,OAAO,EAAE,WAAW,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAE1E,MAAM,WAAW,GAAG,GAAG,CAAC;AACxB,MAAM,qBAAqB,GAAG,CAAC,CAAC;AAEhC,KAAK,UAAU,SAAS,CACtB,KAAe,EACf,IAAY,EACZ,KAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;QAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACrD,4FAA4F;IAC5F,wEAAwE;IACxE,IAAI,GAAG,KAAK,MAAM,EAAE,CAAC;QACnB,MAAM,EAAE,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAC1D,MAAM,IAAI,iBAAiB,CACzB,sBAAsB,IAAI,KAAK,GAAG,OAAO,MAAM,aAAa,KAAK,UAAU,EAAE,EAAE,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,GAAI,EAAuB,CAAC,IAAI,QAAQ,CAAC,CAAC,CAAC,SAAS,GAAG,CACzJ,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,CAAC,CAAC,CAAC;IAC1C,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,CAAC;IAChC,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,+FAA+F;AAC/F,MAAM,OAAO,iBAAkB,SAAQ,KAAK;IAC1C,IAAI,GAAG,mBAAmB,CAAC;CAC5B;AAaD,kFAAkF;AAClF,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAe,EACf,IAAY;AACZ,oFAAoF;AACpF,KAAmE;IAEnE,IAAI,KAAK;QAAE,OAAO,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IAClE,MAAM,IAAI,GACR,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,KAAK,MAAM;QACxC,CAAC,CAAE,CAAC,MAAM,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAsB,CAAC,IAAI;QACtD,CAAC,CAAC,CAAC,CAAC;IACR,IAAI,IAAI,GAAG,WAAW,GAAG,CAAC,GAAG,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,eAAe,CAAC,CAAC;IAC9E,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,EAAE,WAAW,CAAC,CAAC;IAC1D,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IACvE,IAAI,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,MAAM;QAC1D,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,aAAa,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAc;QACxB,KAAK,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;QACtB,QAAQ,EAAE,EAAE,CAAC,SAAS,CAAC,CAAC,CAAC;QACzB,MAAM,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5B,OAAO,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;QACrC,gBAAgB,EAAE,EAAE,CAAC,YAAY,CAAC,EAAE,CAAC;KACtC,CAAC;IACF,qFAAqF;IACrF,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,EAAE,EAAE,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC;IACtF,MAAM,GAAG,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,GAAG,EAAE,GAAG,MAAM,EAAE,MAAM,CAAC,CAAC;IACrE,MAAM,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAClD,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,MAAM,GAAG,GAAG,EAAE;QAClB,IAAI,CAAC,GAAG,CAAC,CAAC;QACV,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,CAAS,CAAC;QACd,GAAG,CAAC;YACF,CAAC,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YACb,CAAC,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC;YAC7B,KAAK,IAAI,CAAC,CAAC;QACb,CAAC,QAAQ,CAAC,GAAG,IAAI,EAAE;QACnB,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;IACF,SAAS,CAAC;QACR,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC;QACtB,IAAI,IAAI,KAAK,CAAC;YAAE,MAAM;QACtB,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;QACxB,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IACD,OAAO,OAAO,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAED,SAAS,OAAO,CACd,KAAe,EACf,IAAY,EACZ,MAAiB,EACjB,KAAoC;IAEpC,MAAM,UAAU,GAAG,CAAC,CAAa,EAAE,EAAE;QACnC,MAAM,CAAC,GAAG,IAAI,QAAQ,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,qBAAqB;YACxC,OAAO,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,EAAE,EAAE,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC7E,OAAO,kBAAkB,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5D,CAAC,CAAC;IACF,OAAO;QACL,IAAI;QACJ,MAAM;QACN,KAAK;QACL,KAAK,CAAC,QAAQ,CAAC,IAAI;YACjB,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC3B,IAAI,CAAC,EAAE;gBAAE,OAAO,SAAS,CAAC;YAC1B,OAAO,UAAU,CAAC,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAChE,CAAC;QACD,KAAK,CAAC,SAAS,CAAC,KAAK;YACnB,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,OAAO,GAAG,CAAC;YAC/B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAqB,CAAC,CAAC;YAClE,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACvD,MAAM,CAAC,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,GAAG,KAAK,CAAC,CAAC;YAC3D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE;gBACnC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAC7F,OAAO,GAAG,CAAC;QACb,CAAC;KACF,CAAC;AACJ,CAAC;AAyBD;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAC,KAAe,EAAE,KAAe;IAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3E,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACnB,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CACvF,CAAC;IACF,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACtB,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,EAAE;YAAE,UAAU,GAAG,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IACH,IAAI,UAAU,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE;QACnC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,GAAG,EAAE;YAC9D,MAAM,IAAI,KAAK,CAAC,mBAAmB,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACxD,IAAI,MAAM,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAoB,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,KAAK;QAAE,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE;YAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IAC7E,MAAM,IAAI,GAAc;QACtB,QAAQ,EAAE,MAAM,CAAC,QAAQ;QACzB,SAAS,EAAE,MAAM,CAAC,MAAM;QACxB,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,KAAK,EAAE,KAAK;QACZ,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI;QACvC,KAAK,CAAC,QAAQ,CAAC,IAAI;YACjB,MAAM,IAAI,GAAG,IAAI,IAAI,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YACxF,OAAO,IAAI,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK;YACvB,MAAM,MAAM,GAAG,IAAI,GAAG,EAAsB,CAAC;YAC7C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAsB,CAAC;YAC1C,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,IAAI,GAAG,KAAK,IAAI,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;gBAChE,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;gBACvB,IAAI,CAAC,CAAC;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;;oBAC7C,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YACpD,CAAC;YACD,KAAK,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,MAAM;gBAAE,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC;oBAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YAC9F,OAAO,GAAG,CAAC;QACb,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,IAAI,EAAE,KAAK;YACtB,MAAM,CAAC,GAAG,MAAM,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;YACjD,IAAI,CAAC,CAAC,MAAM,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,GAAG,EAAE;gBACxC,MAAM,IAAI,KAAK,CAAC,QAAQ,IAAI,yBAAyB,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YACvE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACd,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,EAAE;gBAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;YACtD,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC;YAClB,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;YAC/B,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC;YAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,IAAI,KAAK;gBAAE,IAAI,IAAI,GAAG,IAAI,CAAC,SAAS;oBAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAC5E,CAAC;KACF,CAAC;IACF,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,eAAe,CACpC,MAAiB,EACjB,KAA0C,EAC1C,iBAAiB,GAAG,EAAE;AACtB,0FAA0F;AAC1F,OAAkD;IAElD,2FAA2F;IAC3F,sDAAsD;IACtD,KAAK,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE,GAAG,GAAG,IAAI,CAAC,CAAC;AACvF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,QAAQ,CAAC,GAA8B,EAAE,IAAY;IACnE,IAAI,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC;gBAAE,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACtC,MAAM,KAAK,CAAC;YACZ,SAAS;QACX,CAAC;QACD,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC5B,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACtB,GAAG,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAC;YAC3B,CAAC,GAAG,CAAC,CAAC;QACR,CAAC;QACD,GAAG,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAClB,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC;IACpB,CAAC;IACD,IAAI,CAAC;QAAE,MAAM,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;AAC/B,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,YAAY,CAC1B,MAAiB,EACjB,KAA0C,EAC1C,iBAAyB,EACzB,OAAkD;IAElD,MAAM,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC;IACxB,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,WAAW,CAAC,CAAC;IACzC,MAAM,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACpC,IAAI,CAAC,GAAG,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,GACT,MAAM,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,gBAAgB,KAAK,EAAE,IAAI,iBAAiB,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvF,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACtB,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC,CAAC,SAAS,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;IACnC,CAAC,CAAC,WAAW,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;IAC5C,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,gBAAgB,CAAC,CAAC;IAC5C,CAAC,CAAC,YAAY,CAAC,EAAE,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC;IACxC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACjB,MAAM,IAAI,CAAC;IACX,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,CAAC,CAAS,EAAE,EAAE;QAC5B,OAAO,CAAC,IAAI,IAAI;YAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;QAC3E,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChB,CAAC,CAAC;IACF,IAAI,MAAM,GAAG,WAAW,CAAC;IACzB,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,KAAK,EAAE,CAAC;QACvC,IAAI,IAAI,KAAK,IAAI;YAAE,SAAS;QAC5B,IAAI,IAAI,IAAI,IAAI;YAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;QAC7E,IAAI,GAAG,IAAI,CAAC;QACZ,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;QACrC,MAAM,EAAE,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;QACnC,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QACtB,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,qBAAqB,CAAC,CAAC;QACvC,EAAE,CAAC,SAAS,CAAC,CAAC,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC;QACnC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,8DAA8D;QAC3F,OAAO,CAAC,IAAI,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;QAChE,OAAO,EAAE,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;QAC3D,MAAM,IAAI,EAAE,GAAG,UAAU,CAAC,MAAM,CAAC;QACjC,MAAM,EAAE,CAAC;QACT,MAAM,UAAU,CAAC;IACnB,CAAC;IACD,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,CAAC,CAAC,CAAC,CAAC;IACX,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnC,MAAM,IAAI,GAAG,IAAI,UAAU,CAAC,EAAE,CAAC,CAAC;IAChC,MAAM,EAAE,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACrC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;IACvC,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC;IACtC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,EAAE,aAAa,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;IACvE,MAAM,GAAG,CAAC;IACV,MAAM,GAAG,CAAC;IACV,MAAM,IAAI,CAAC;IACX,MAAM,GAAG,CAAC;AACZ,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,IAAe,EACf,KAAe,EACf,IAAY,EACZ,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE;IAEtB,uFAAuF;IACvF,KAAK,SAAS,CAAC,CAAC,KAAK;QACnB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,IAAI,IAAI,GAAG,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC1C,KAAK,IAAI,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,IAAI,GAAG,GAAG,IAAI,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE;gBACvE,MAAM,CAAC,IAAI,EAAE,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IACD,MAAM,KAAK,CAAC,KAAK,CACf,IAAI,EACJ,eAAe,CACb;QACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,SAAS;QACtB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,SAAS;QACT,gBAAgB,EAAE,EAAE;KACrB,EACD,KAAK,EAAE,CACR,CACF,CAAC;AACJ,CAAC;AAED,2FAA2F;AAC3F,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAAe,EACf,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE;IAEtB,MAAM,MAAM,GAAiB,EAAE,CAAC;IAChC,KAAK,SAAS,CAAC,CAAC,KAAK;QACnB,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE;YAAE,MAAM,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7F,CAAC;IACD,IAAI,KAAK,EAAE,MAAM,CAAC,IAAI,eAAe,CACnC;QACE,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,SAAS;QACtB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,SAAS;QACT,gBAAgB,EAAE,EAAE;KACrB,EACD,KAAK,EAAE,CACR;QACC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjB,MAAM,GAAG,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACrE,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC;IACvD,OAAO,GAAG,CAAC;AACb,CAAC","sourcesContent":["/**\n * Page-level random access to a cell kept as celld keeps it: LTX files in a bucket\n * (`…/ltx/e<epoch>/<level>/<min>-<max>.ltx`), a snapshot (TXID 1..N) and the incremental files\n * after it. Nothing is read whole: an LTX file ends with a page index (pgno → offset, size), so\n * opening a file reads its 100-byte header and its index, and a page is ONE range read plus an\n * LZ4 decode. The newest file holding a page wins; `compact` folds them into a new snapshot.\n *\n * This is the read side a SQLite VFS needs to serve a database from any FilesApi (OPFS, File\n * System Access, IndexedDB, S3…) without loading it: xRead → `readPage` (through a page cache);\n * the write side is the existing LTX log (one incremental file per commit).\n */\nimport type { FilesApi } from \"@statewalker/webrun-files\";\nimport { Crc64 } from \"./crc64.js\";\nimport { CHECKSUM_FLAG, type LtxHeader, lockPgno } from \"./ltx.js\";\nimport { lz4Compress, lz4Decompress, lz4FrameDecompress } from \"./lz4.js\";\n\nconst HEADER_SIZE = 100;\nconst PAGE_HEADER_FLAG_SIZE = 1;\n\nasync function readRange(\n  files: FilesApi,\n  path: string,\n  start: number,\n  length: number,\n): Promise<Uint8Array> {\n  const chunks: Uint8Array[] = [];\n  for await (const c of files.read(path, { start, length })) chunks.push(c);\n  const got = chunks.reduce((n, c) => n + c.length, 0);\n  // A short read is a file that changed or went away under its reader: say which, rather than\n  // decode past its end (\"Offset is outside the bounds of the DataView\").\n  if (got !== length) {\n    const st = await files.stats(path).catch(() => undefined);\n    throw new LtxShortReadError(\n      `ltx: short read of ${path}: ${got} of ${length} bytes at ${start} (file ${st?.kind === \"file\" ? `${(st as { size: number }).size} bytes` : \"missing\"})`,\n    );\n  }\n  if (chunks.length === 1) return chunks[0];\n  const out = new Uint8Array(got);\n  let o = 0;\n  for (const c of chunks) out.set(c, o), (o += c.length);\n  return out;\n}\n\n/** A range read that got fewer bytes than asked: the file changed or was removed meanwhile. */\nexport class LtxShortReadError extends Error {\n  name = \"LtxShortReadError\";\n}\n\nexport interface LtxPages {\n  path: string;\n  header: LtxHeader;\n  /** pgno → [offset of the page's 6-byte header, size including it]. */\n  index: Map<number, [number, number]>;\n  /** One range read and an LZ4 decode. */\n  readPage(pgno: number): Promise<Uint8Array | undefined>;\n  /** Several pages of this file with ONE range read (they are stored in pgno order). */\n  readPages(pgnos: number[]): Promise<Map<number, Uint8Array>>;\n}\n\n/** Opens an LTX file for page reads: reads its header and its page index only. */\nexport async function openLtxPages(\n  files: FilesApi,\n  path: string,\n  /** The header and index of a file this process just wrote: nothing is read back. */\n  known?: { header: LtxHeader; index: Map<number, [number, number]> },\n): Promise<LtxPages> {\n  if (known) return pagesOf(files, path, known.header, known.index);\n  const size =\n    (await files.stats(path))?.kind === \"file\"\n      ? ((await files.stats(path)) as { size: number }).size\n      : 0;\n  if (size < HEADER_SIZE + 6 + 24) throw new Error(`ltx: ${path} is too small`);\n  const head = await readRange(files, path, 0, HEADER_SIZE);\n  const hv = new DataView(head.buffer, head.byteOffset, head.byteLength);\n  if (new TextDecoder().decode(head.subarray(0, 4)) !== \"LTX1\")\n    throw new Error(`ltx: ${path}: bad magic`);\n  const header: LtxHeader = {\n    flags: hv.getUint32(4),\n    pageSize: hv.getUint32(8),\n    commit: hv.getUint32(12),\n    minTXID: hv.getBigUint64(16),\n    maxTXID: hv.getBigUint64(24),\n    timestamp: Number(hv.getBigInt64(32)),\n    preApplyChecksum: hv.getBigUint64(40),\n  };\n  // Trailer: … index | u64 index length | u64 post-apply checksum | u64 file checksum.\n  const tail = await readRange(files, path, size - 24, 24);\n  const idxLen = Number(new DataView(tail.buffer, tail.byteOffset, 24).getBigUint64(0));\n  const idx = await readRange(files, path, size - 24 - idxLen, idxLen);\n  const index = new Map<number, [number, number]>();\n  let o = 0;\n  const varint = () => {\n    let x = 0;\n    let shift = 0;\n    let b: number;\n    do {\n      b = idx[o++];\n      x += (b & 0x7f) * 2 ** shift;\n      shift += 7;\n    } while (b & 0x80);\n    return x;\n  };\n  for (;;) {\n    const pgno = varint();\n    if (pgno === 0) break;\n    const offset = varint();\n    index.set(pgno, [offset, varint()]);\n  }\n  return pagesOf(files, path, header, index);\n}\n\nfunction pagesOf(\n  files: FilesApi,\n  path: string,\n  header: LtxHeader,\n  index: Map<number, [number, number]>,\n): LtxPages {\n  const decodePage = (b: Uint8Array) => {\n    const v = new DataView(b.buffer, b.byteOffset, b.byteLength);\n    if (v.getUint16(4) & PAGE_HEADER_FLAG_SIZE)\n      return lz4Decompress(b.subarray(10, 10 + v.getUint32(6)), header.pageSize);\n    return lz4FrameDecompress(b.subarray(6), header.pageSize);\n  };\n  return {\n    path,\n    header,\n    index,\n    async readPage(pgno) {\n      const at = index.get(pgno);\n      if (!at) return undefined;\n      return decodePage(await readRange(files, path, at[0], at[1]));\n    },\n    async readPages(pgnos) {\n      const out = new Map<number, Uint8Array>();\n      const wanted = pgnos.filter((p) => index.has(p));\n      if (!wanted.length) return out;\n      const spans = wanted.map((p) => index.get(p) as [number, number]);\n      const start = Math.min(...spans.map((s) => s[0]));\n      const end = Math.max(...spans.map((s) => s[0] + s[1]));\n      const b = await readRange(files, path, start, end - start);\n      for (const [i, p] of wanted.entries())\n        out.set(p, decodePage(b.subarray(spans[i][0] - start, spans[i][0] - start + spans[i][1])));\n      return out;\n    },\n  };\n}\n\nexport interface PagedCell {\n  pageSize: number;\n  /** Database size in pages, after the newest file. */\n  pageCount: number;\n  maxTXID: bigint;\n  /** The files, oldest first (the snapshot, then the incremental ones). */\n  files: LtxPages[];\n  /** The page as of the newest file holding it (zeros past the database end). */\n  readPage(pgno: number): Promise<Uint8Array>;\n  /** Pages `from`..`from+count-1` (within the database), one range read per file they live in. */\n  readRun(from: number, count: number): Promise<Map<number, Uint8Array>>;\n  /** Which file serves each page (for tests and traces). */\n  source(pgno: number): string | undefined;\n  /**\n   * Adds the cell's next file (the one just written after `maxTXID`): reads its header and index\n   * only, so a commit costs one file whatever the number of files before it.\n   */\n  append(\n    path: string,\n    known?: { header: LtxHeader; index: Map<number, [number, number]> },\n  ): Promise<void>;\n}\n\n/**\n * A cell's pages over its LTX files: the newest snapshot (min TXID 1) and the files after it,\n * in TXID order. Only headers and page indexes are read.\n */\nexport async function openPagedCell(files: FilesApi, paths: string[]): Promise<PagedCell> {\n  const opened = await Promise.all(paths.map((p) => openLtxPages(files, p)));\n  opened.sort((a, b) =>\n    a.header.maxTXID < b.header.maxTXID ? -1 : a.header.maxTXID > b.header.maxTXID ? 1 : 0,\n  );\n  let snapshotAt = -1;\n  opened.forEach((f, i) => {\n    if (f.header.minTXID === 1n) snapshotAt = i;\n  });\n  if (snapshotAt < 0) throw new Error(\"ltx: no snapshot among the cell's files\");\n  const chain = opened.slice(snapshotAt);\n  for (let i = 1; i < chain.length; i++)\n    if (chain[i].header.minTXID !== chain[i - 1].header.maxTXID + 1n)\n      throw new Error(`ltx: gap before ${chain[i].path}`);\n  let newest = chain[chain.length - 1].header;\n  const owner = new Map<number, LtxPages>();\n  for (const f of chain) for (const pgno of f.index.keys()) owner.set(pgno, f);\n  const cell: PagedCell = {\n    pageSize: newest.pageSize,\n    pageCount: newest.commit,\n    maxTXID: newest.maxTXID,\n    files: chain,\n    source: (pgno) => owner.get(pgno)?.path,\n    async readPage(pgno) {\n      const page = pgno <= cell.pageCount ? await owner.get(pgno)?.readPage(pgno) : undefined;\n      return page ?? new Uint8Array(cell.pageSize);\n    },\n    async readRun(from, count) {\n      const byFile = new Map<LtxPages, number[]>();\n      const out = new Map<number, Uint8Array>();\n      for (let p = from; p < from + count && p <= cell.pageCount; p++) {\n        const f = owner.get(p);\n        if (!f) out.set(p, new Uint8Array(cell.pageSize));\n        else byFile.set(f, [...(byFile.get(f) ?? []), p]);\n      }\n      for (const [f, pgnos] of byFile) for (const [p, d] of await f.readPages(pgnos)) out.set(p, d);\n      return out;\n    },\n    async append(path, known) {\n      const f = await openLtxPages(files, path, known);\n      if (f.header.minTXID !== cell.maxTXID + 1n)\n        throw new Error(`ltx: ${path} does not follow TXID ${cell.maxTXID}`);\n      chain.push(f);\n      for (const pgno of f.index.keys()) owner.set(pgno, f);\n      newest = f.header;\n      cell.pageCount = newest.commit;\n      cell.maxTXID = newest.maxTXID;\n      for (const [pgno] of owner) if (pgno > cell.pageCount) owner.delete(pgno);\n    },\n  };\n  return cell;\n}\n\n/**\n * An LTX file produced page by page, as celld writes it (block format: size-prefixed LZ4 page\n * blocks, page index, trailer, CRC-64/ISO over the uncompressed pages): the bytes come out as an\n * async stream while `pages` is consumed, so a writer holds one page at a time plus the index\n * (a few bytes per page), never the database.\n */\nexport async function* encodeLtxStream(\n  header: LtxHeader,\n  pages: AsyncIterable<[number, Uint8Array]>,\n  postApplyChecksum = 0n,\n  /** Filled while encoding: the file's page index (what `openLtxPages` would read back). */\n  written?: { index: Map<number, [number, number]> },\n): AsyncGenerator<Uint8Array> {\n  // Coalesced into 256 KiB chunks: a writer pays per call (an OPFS SyncAccessHandle write, a\n  // stream write), and a page makes three small pieces.\n  yield* coalesce(encodePieces(header, pages, postApplyChecksum, written), 256 * 1024);\n}\n\nasync function* coalesce(src: AsyncIterable<Uint8Array>, size: number): AsyncGenerator<Uint8Array> {\n  let buf = new Uint8Array(size);\n  let n = 0;\n  for await (const piece of src) {\n    if (piece.length >= size) {\n      if (n) yield buf.slice(0, n), (n = 0);\n      yield piece;\n      continue;\n    }\n    if (n + piece.length > size) {\n      yield buf.slice(0, n);\n      buf = new Uint8Array(size);\n      n = 0;\n    }\n    buf.set(piece, n);\n    n += piece.length;\n  }\n  if (n) yield buf.slice(0, n);\n}\n\nasync function* encodePieces(\n  header: LtxHeader,\n  pages: AsyncIterable<[number, Uint8Array]>,\n  postApplyChecksum: bigint,\n  written?: { index: Map<number, [number, number]> },\n): AsyncGenerator<Uint8Array> {\n  const crc = new Crc64();\n  const head = new Uint8Array(HEADER_SIZE);\n  const v = new DataView(head.buffer);\n  head.set(new TextEncoder().encode(\"LTX1\"), 0);\n  const flags =\n    header.flags ?? (header.preApplyChecksum === 0n && postApplyChecksum === 0n ? 2 : 0);\n  v.setUint32(4, flags);\n  v.setUint32(8, header.pageSize);\n  v.setUint32(12, header.commit);\n  v.setBigUint64(16, header.minTXID);\n  v.setBigUint64(24, header.maxTXID);\n  v.setBigInt64(32, BigInt(header.timestamp));\n  v.setBigUint64(40, header.preApplyChecksum);\n  v.setBigUint64(72, header.nodeID ?? 0n);\n  crc.update(head);\n  yield head;\n  const index: number[] = [];\n  const uvarint = (n: number) => {\n    while (n >= 0x80) index.push((n & 0x7f) | 0x80), (n = Math.floor(n / 128));\n    index.push(n);\n  };\n  let offset = HEADER_SIZE;\n  const lock = lockPgno(header.pageSize);\n  let last = 0;\n  for await (const [pgno, data] of pages) {\n    if (pgno === lock) continue;\n    if (pgno <= last) throw new Error(\"ltx: pages must come in ascending order\");\n    last = pgno;\n    const compressed = lz4Compress(data);\n    const ph = new Uint8Array(10);\n    const pv = new DataView(ph.buffer);\n    pv.setUint32(0, pgno);\n    pv.setUint16(4, PAGE_HEADER_FLAG_SIZE);\n    pv.setUint32(6, compressed.length);\n    crc.update(ph).update(data); // the checksum covers uncompressed pages (encodeLtx's `feed`)\n    uvarint(pgno), uvarint(offset), uvarint(10 + compressed.length);\n    written?.index.set(pgno, [offset, 10 + compressed.length]);\n    offset += 10 + compressed.length;\n    yield ph;\n    yield compressed;\n  }\n  const end = new Uint8Array(6);\n  uvarint(0);\n  const idx = Uint8Array.from(index);\n  const tail = new Uint8Array(16);\n  const tv = new DataView(tail.buffer);\n  tv.setBigUint64(0, BigInt(idx.length));\n  tv.setBigUint64(8, postApplyChecksum);\n  crc.update(end).update(idx).update(tail);\n  const sum = new Uint8Array(8);\n  new DataView(sum.buffer).setBigUint64(0, CHECKSUM_FLAG | crc.digest());\n  yield end;\n  yield idx;\n  yield tail;\n  yield sum;\n}\n\n/**\n * Folds a cell's files into one snapshot (TXID 1..maxTXID) written to `files` at `path`, page by\n * page: each page is read by range from the file that holds it and streamed into the new file,\n * so memory stays at one page (plus the index), whatever the database size. The caller removes\n * the files it replaces once this resolves.\n */\nexport async function compactPagedCellTo(\n  cell: PagedCell,\n  files: FilesApi,\n  path: string,\n  timestamp = Date.now(),\n): Promise<void> {\n  // In runs of 256 pages: one range read per file a run touches (not one read per page).\n  async function* pages(): AsyncGenerator<[number, Uint8Array]> {\n    for (let from = 1; from <= cell.pageCount; from += 256) {\n      const run = await cell.readRun(from, 256);\n      for (let pgno = from; pgno < from + 256 && pgno <= cell.pageCount; pgno++)\n        yield [pgno, run.get(pgno) ?? new Uint8Array(cell.pageSize)];\n    }\n  }\n  await files.write(\n    path,\n    encodeLtxStream(\n      {\n        pageSize: cell.pageSize,\n        commit: cell.pageCount,\n        minTXID: 1n,\n        maxTXID: cell.maxTXID,\n        timestamp,\n        preApplyChecksum: 0n,\n      },\n      pages(),\n    ),\n  );\n}\n\n/** The compacted snapshot as bytes (small cells, tests); compactPagedCellTo streams it. */\nexport async function compactPagedCell(\n  cell: PagedCell,\n  timestamp = Date.now(),\n): Promise<Uint8Array> {\n  const chunks: Uint8Array[] = [];\n  async function* pages(): AsyncGenerator<[number, Uint8Array]> {\n    for (let pgno = 1; pgno <= cell.pageCount; pgno++) yield [pgno, await cell.readPage(pgno)];\n  }\n  for await (const c of encodeLtxStream(\n    {\n      pageSize: cell.pageSize,\n      commit: cell.pageCount,\n      minTXID: 1n,\n      maxTXID: cell.maxTXID,\n      timestamp,\n      preApplyChecksum: 0n,\n    },\n    pages(),\n  ))\n    chunks.push(c);\n  const out = new Uint8Array(chunks.reduce((n, c) => n + c.length, 0));\n  let o = 0;\n  for (const c of chunks) out.set(c, o), (o += c.length);\n  return out;\n}\n"]}

package/dist/ltx/ltx-store-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/ltx/ltx-store-worker.js

@@ -0,0 +1,38 @@
+/// <reference lib="webworker" />
+/**
+ * The LTX store as a worker of its own (spawned by a shard worker that runs cells on the LTX
+ * VFS): cells' LTX files in an OPFS directory, in celld's layout. The shard connects with
+ * `{ op: "init", sab, dir, compactAfter }`, then sends the VFS's requests; answers go into
+ * the shared buffer.
+ */
+import { BrowserFilesApi } from "@statewalker/webrun-files-browser";
+import { MemFilesApi } from "@statewalker/webrun-files-mem";
+import { ltxCellStore } from "./ltx-page-server.js";
+import { opfsSyncFiles } from "./opfs-sync-files.js";
+let store;
+let ready;
+self.onmessage = (e) => {
+    const m = e.data;
+    if (m?.op === "init") {
+        ready = (async () => {
+            // The cells' FilesApi, by name (an instance cannot cross to this worker): "opfs-sync"
+            // (default: synchronous handles; each file written under a temporary name and moved into
+            // place, so a file being written is never read), "opfs" (OPFS's asynchronous API),
+            // "memory" (nothing kept).
+            const files = m.files === "memory"
+                ? new MemFilesApi()
+                : await (async () => {
+                    const root = await navigator.storage.getDirectory();
+                    const dir = await root.getDirectoryHandle(m.dir, { create: true });
+                    return m.files === "opfs"
+                        ? new BrowserFilesApi({ rootHandle: dir })
+                        : opfsSyncFiles(dir);
+                })();
+            store = ltxCellStore(files, m.sab, { compactAfter: m.compactAfter, lockScope: m.dir });
+            postMessage({ op: "ready" });
+        })();
+        return;
+    }
+    void ready?.then(() => store?.handle(m));
+};
+//# sourceMappingURL=ltx-store-worker.js.map

package/dist/ltx/ltx-store-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ltx-store-worker.js","sourceRoot":"","sources":["../../src/ltx/ltx-store-worker.ts"],"names":[],"mappings":"AAAA,iCAAiC;AACjC;;;;;GAKG;AACH,OAAO,EAAE,eAAe,EAAE,MAAM,mCAAmC,CAAC;AACpE,OAAO,EAAE,WAAW,EAAE,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,sBAAsB,CAAC;AAErD,IAAI,KAAkD,CAAC;AACvD,IAAI,KAAgC,CAAC;AACrC,IAAI,CAAC,SAAS,GAAG,CAAC,CAAe,EAAE,EAAE;IACnC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;IACjB,IAAI,CAAC,EAAE,EAAE,KAAK,MAAM,EAAE,CAAC;QACrB,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE;YAClB,sFAAsF;YACtF,yFAAyF;YACzF,mFAAmF;YACnF,2BAA2B;YAC3B,MAAM,KAAK,GACT,CAAC,CAAC,KAAK,KAAK,QAAQ;gBAClB,CAAC,CAAC,IAAI,WAAW,EAAE;gBACnB,CAAC,CAAC,MAAM,CAAC,KAAK,IAAI,EAAE;oBAChB,MAAM,IAAI,GAAG,MAAM,SAAS,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;oBACpD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;oBACnE,OAAO,CAAC,CAAC,KAAK,KAAK,MAAM;wBACvB,CAAC,CAAC,IAAI,eAAe,CAAC,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;wBAC1C,CAAC,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;gBACzB,CAAC,CAAC,EAAE,CAAC;YACX,KAAK,GAAG,YAAY,CAAC,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,SAAS,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;YACvF,WAAW,CAAC,EAAE,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC;QAC/B,CAAC,CAAC,EAAE,CAAC;QACL,OAAO;IACT,CAAC;IACD,KAAK,KAAK,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;AAC3C,CAAC,CAAC","sourcesContent":["/// <reference lib=\"webworker\" />\n/**\n * The LTX store as a worker of its own (spawned by a shard worker that runs cells on the LTX\n * VFS): cells' LTX files in an OPFS directory, in celld's layout. The shard connects with\n * `{ op: \"init\", sab, dir, compactAfter }`, then sends the VFS's requests; answers go into\n * the shared buffer.\n */\nimport { BrowserFilesApi } from \"@statewalker/webrun-files-browser\";\nimport { MemFilesApi } from \"@statewalker/webrun-files-mem\";\nimport { ltxCellStore } from \"./ltx-page-server.js\";\nimport { opfsSyncFiles } from \"./opfs-sync-files.js\";\n\nlet store: ReturnType<typeof ltxCellStore> | undefined;\nlet ready: Promise<void> | undefined;\nself.onmessage = (e: MessageEvent) => {\n  const m = e.data;\n  if (m?.op === \"init\") {\n    ready = (async () => {\n      // The cells' FilesApi, by name (an instance cannot cross to this worker): \"opfs-sync\"\n      // (default: synchronous handles; each file written under a temporary name and moved into\n      // place, so a file being written is never read), \"opfs\" (OPFS's asynchronous API),\n      // \"memory\" (nothing kept).\n      const files =\n        m.files === \"memory\"\n          ? new MemFilesApi()\n          : await (async () => {\n              const root = await navigator.storage.getDirectory();\n              const dir = await root.getDirectoryHandle(m.dir, { create: true });\n              return m.files === \"opfs\"\n                ? new BrowserFilesApi({ rootHandle: dir })\n                : opfsSyncFiles(dir);\n            })();\n      store = ltxCellStore(files, m.sab, { compactAfter: m.compactAfter, lockScope: m.dir });\n      postMessage({ op: \"ready\" });\n    })();\n    return;\n  }\n  void ready?.then(() => store?.handle(m));\n};\n"]}

package/dist/ltx/ltx.d.ts

@@ -0,0 +1,45 @@
+/**
+ * LTX v3 (superfly/ltx v0.5.2), byte for byte as celld writes it — see
+ * crates/ltx/reference/ltx-format.md in denoland/celld. Big-endian integers, 100-byte
+ * header, size-prefixed LZ4 page blocks, page index, 16-byte trailer, CRC-64/ISO checksums
+ * with the top bit always set.
+ */
+export declare const CHECKSUM_FLAG: bigint;
+/** Header flag: the file carries no rolling database checksums (what celld 0.6.2 writes). */
+export declare const HEADER_FLAG_NO_CHECKSUM: number;
+export interface LtxHeader {
+    /** Header flags; defaults to NO_CHECKSUM when both rolling checksums are 0 (celld's files). */
+    flags?: number;
+    /** WAL position the file was cut from (litestream WAL mode; 0 in journal mode). */
+    walOffset?: bigint;
+    walSize?: bigint;
+    walSalt1?: number;
+    walSalt2?: number;
+    pageSize: number;
+    /** Database size after the transaction, in pages. */
+    commit: number;
+    minTXID: bigint;
+    maxTXID: bigint;
+    timestamp: number;
+    preApplyChecksum: bigint;
+    nodeID?: bigint;
+}
+export interface LtxFile {
+    header: LtxHeader;
+    /** pgno → page bytes, ascending. */
+    pages: Map<number, Uint8Array>;
+    postApplyChecksum: bigint;
+}
+/** The SQLite lock page (never stored, excluded from checksums). */
+export declare const lockPgno: (pageSize: number) => number;
+export declare const checksumPage: (pgno: number, data: Uint8Array) => bigint;
+/** Rolling database checksum over pages 1..n (lock page excluded). */
+export declare function databaseChecksum(pageSize: number, pageAt: (pgno: number) => Uint8Array, n: number): bigint;
+/**
+ * `format: "legacy"` (default) writes one LZ4 frame per page, as celld 0.6.2 does; `"block"`
+ * writes v0.5.2's size-prefixed blocks (celld's decoder accepts both).
+ */
+export declare function encodeLtx(file: LtxFile, format?: "legacy" | "block"): Uint8Array;
+export declare function decodeLtx(bytes: Uint8Array): LtxFile;
+/** `<minTXID:016x>-<maxTXID:016x>.ltx` */
+export declare const ltxFileName: (min: bigint, max: bigint) => string;