@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/dist/ltx/lz4.js

@@ -0,0 +1,264 @@
+/**
+ * LZ4 block-codec adapter.
+ *
+ * The LTX format uses LZ4 **block** format (not frame format), matching Go's
+ * `github.com/pierrec/lz4/v4` `CompressBlock`/`UncompressBlock`.
+ *
+ * `compressBlock` is a direct TypeScript port of `pierrec/lz4/v4`'s internal
+ * `lz4block.Compressor.CompressBlock` (the default Fast compressor used by
+ * litestream). It is **byte-identical** to Go's output for the same input, so
+ * LTX files we write match Go's `superfly/ltx` encoder exactly (validated
+ * against the Go fixtures in `test/ltx/go-interop.test.ts` and CI's
+ * `interop-go` job).
+ *
+ * `decompressBlock`/`decompressBlockInto` delegate to the pure-JS `lz4js`
+ * implementation, which decodes any conforming LZ4 block (decompression does
+ * not need to be byte-identical).
+ */
+import lz4 from 'lz4js';
+import { LTXFormatError } from "../errors.js";
+// Compressor constants, mirrored from `internal/lz4block/block.go`.
+const HASH_LOG = 16;
+const HT_SIZE = 1 << HASH_LOG; // 65536 hash-table entries.
+const WIN_SIZE = 1 << 16; // 64 KiB window.
+const WIN_MASK = WIN_SIZE - 1;
+const MIN_MATCH = 4;
+const MF_LIMIT = 10 + MIN_MATCH; // Last match cannot start within the last 14 bytes.
+const ADAPT_SKIP_LOG = 7;
+const PRIME6 = 227718039650203n;
+const U64_MASK = 0xffffffffffffffffn;
+const U32_MASK = 0xffffffffn;
+// Hash table state. `inUse` is a bitmap so we can clear the table cheaply.
+const hashTable = new Uint16Array(HT_SIZE);
+const inUse = new Uint32Array(HT_SIZE / 32);
+/**
+ * compressBlockBound returns the maximum compressed size for `n` input bytes
+ * using the Fast compressor (`n + n/255 + 16`).
+ */
+export function compressBlockBound(n) {
+    return n + ((n / 255) | 0) + 16;
+}
+/** Hashes the lower 6 bytes of a 64-bit little-endian value into a hash index. */
+function blockHash(x) {
+    return Number((((x << 16n) * PRIME6) & U64_MASK) >> 48n);
+}
+/** Reads the 64-bit little-endian value at `o` (used by the Fast scan). */
+function readU64(b, o) {
+    let x = 0n;
+    for (let i = 7; i >= 0; i--)
+        x = (x << 8n) | BigInt(b[o + i]);
+    return x;
+}
+/** Reads the 32-bit little-endian value at `o`. */
+function readU32(b, o) {
+    return (b[o] | (b[o + 1] << 8) | (b[o + 2] << 16) | (b[o + 3] << 24)) >>> 0;
+}
+/** Returns the number of leading zero bytes in a 64-bit little-endian value. */
+function trailingZeroBytes(x) {
+    let n = 0;
+    while ((x & 0xffn) === 0n && n < 8) {
+        x >>= 8n;
+        n++;
+    }
+    return n;
+}
+/**
+ * Look up the two 16-bit halves of a stored match offset for a hash. Mirrors
+ * `Compressor.get`: the table stores 16-bit offsets relative to a 64 KiB block
+ * boundary. Returns a position that may be before the start of the window (in
+ * which case `offset >= WIN_SIZE` short-circuits the candidate check).
+ */
+function tableGet(h, si) {
+    h &= HT_SIZE - 1;
+    let i = 0;
+    if (inUse[h >>> 5] & (1 << (h & 31)))
+        i = hashTable[h];
+    i += si & ~WIN_MASK;
+    if (i >= si)
+        i -= WIN_SIZE;
+    return i;
+}
+function tablePut(h, si) {
+    h &= HT_SIZE - 1;
+    hashTable[h] = si & 0xffff;
+    inUse[h >>> 5] |= 1 << (h & 31);
+}
+/**
+ * Compresses `src` into `dst` as a Fast-mode LZ4 block, writing at offset 0.
+ * Mirrors `lz4block.Compressor.CompressBlock`. Returns the number of bytes
+ * written, or 0 if `dst` is too small (which happens only when the caller does
+ * not size `dst` with `compressBlockBound`).
+ */
+function compressBlockInto(src, dst) {
+    // Clear the reused table so output is deterministic (issue #65).
+    inUse.fill(0);
+    // In normal use `dst` is sized by `compressBlockBound`, so the compressor
+    // always emits a valid block (even for incompressible input). An undersized
+    // `dst` is caught by the `di >= dst.length` guards below and returns 0.
+    let si = 0;
+    let di = 0;
+    let anchor = 0;
+    const sn = src.length - MF_LIMIT;
+    if (sn > 0) {
+        for (; si < sn;) {
+            const match = readU64(src, si);
+            let h = blockHash(match);
+            const h2 = blockHash(match >> 8n);
+            const ref = tableGet(h, si);
+            const ref2 = tableGet(h2, si + 1);
+            tablePut(h, si);
+            tablePut(h2, si + 1);
+            let offset = si - ref;
+            if (offset <= 0 || offset >= WIN_SIZE || (match & U32_MASK) !== BigInt(readU32(src, ref))) {
+                h = blockHash(match >> 16n);
+                const ref3 = tableGet(h, si + 2);
+                si += 1;
+                offset = si - ref2;
+                if (offset <= 0 || offset >= WIN_SIZE || ((match >> 8n) & U32_MASK) !== BigInt(readU32(src, ref2))) {
+                    si += 1;
+                    offset = si - ref3;
+                    tablePut(h, si);
+                    if (offset <= 0 || offset >= WIN_SIZE || ((match >> 16n) & U32_MASK) !== BigInt(readU32(src, ref3))) {
+                        si += 2 + ((si - anchor) >>> ADAPT_SKIP_LOG);
+                        continue;
+                    }
+                }
+            }
+            // Match found.
+            let lLen = si - anchor;
+            let mLen = 4;
+            // Extend backwards, reducing literals.
+            let tOff = si - offset - 1;
+            while (lLen > 0 && tOff >= 0 && src[si - 1] === src[tOff]) {
+                si--;
+                tOff--;
+                lLen--;
+                mLen++;
+            }
+            // Continue search at the end of the match. `matchBase` is the position
+            // just past the first 4 matched bytes.
+            const matchBase = si + MIN_MATCH;
+            si = si + mLen;
+            // Find the longest match by comparing 8 bytes at a time.
+            while (si + 8 <= sn) {
+                const x = readU64(src, si) ^ readU64(src, si - offset);
+                if (x === 0n) {
+                    si += 8;
+                }
+                else {
+                    si += trailingZeroBytes(x);
+                    break;
+                }
+            }
+            mLen = si - matchBase;
+            if (di >= dst.length)
+                return 0;
+            if (mLen < 0xf)
+                dst[di] = mLen;
+            else
+                dst[di] = 0xf;
+            // Encode literal length.
+            if (lLen < 0xf) {
+                dst[di] |= lLen << 4;
+            }
+            else {
+                dst[di] |= 0xf0;
+                di++;
+                let l = lLen - 0xf;
+                while (l >= 0xff && di < dst.length) {
+                    dst[di] = 0xff;
+                    di++;
+                    l -= 0xff;
+                }
+                if (di >= dst.length)
+                    return 0;
+                dst[di] = l;
+            }
+            di++;
+            if (di + lLen > dst.length)
+                return 0;
+            dst.set(src.subarray(anchor, anchor + lLen), di);
+            di += lLen + 2;
+            anchor = si;
+            if (di > dst.length)
+                return 0;
+            dst[di - 2] = offset & 0xff;
+            dst[di - 1] = (offset >> 8) & 0xff;
+            // Encode match length part 2.
+            if (mLen >= 0xf) {
+                let m = mLen - 0xf;
+                while (m >= 0xff && di < dst.length) {
+                    dst[di] = 0xff;
+                    di++;
+                    m -= 0xff;
+                }
+                if (di >= dst.length)
+                    return 0;
+                dst[di] = m;
+                di++;
+            }
+            if (si >= sn)
+                break;
+            // Hash match end-2.
+            h = blockHash(readU64(src, si - 2));
+            tablePut(h, si - 2);
+        }
+    }
+    // Last literals.
+    if (di >= dst.length)
+        return 0;
+    const lLen = src.length - anchor;
+    if (lLen < 0xf) {
+        dst[di] = lLen << 4;
+    }
+    else {
+        dst[di] = 0xf0;
+        di++;
+        let l = lLen - 0xf;
+        while (l >= 0xff && di < dst.length) {
+            dst[di] = 0xff;
+            di++;
+            l -= 0xff;
+        }
+        if (di >= dst.length)
+            return 0;
+        dst[di] = l;
+    }
+    di++;
+    if (di + (src.length - anchor) > dst.length)
+        return 0;
+    dst.set(src.subarray(anchor), di);
+    di += src.length - anchor;
+    return di;
+}
+/**
+ * compressBlock compresses `data` as a Fast-mode LZ4 block and returns the
+ * compressed bytes. The result is byte-identical to Go's `pierrec/lz4/v4`
+ * `CompressBlock` for the same input.
+ */
+export function compressBlock(data) {
+    const dst = new Uint8Array(compressBlockBound(data.length));
+    const n = compressBlockInto(data, dst);
+    return dst.slice(0, n);
+}
+/**
+ * decompressBlock decompresses an LZ4 block into a buffer of exactly
+ * `uncompressedSize` bytes. Throws if the block is invalid or produces a
+ * different size.
+ */
+export function decompressBlock(data, uncompressedSize) {
+    const dst = new Uint8Array(uncompressedSize);
+    const n = lz4.decompressBlock(data, dst, 0, data.length, 0);
+    if (n !== uncompressedSize) {
+        throw new LTXFormatError(`lz4 block decompressed to ${n} bytes, expected ${uncompressedSize}`);
+    }
+    return dst;
+}
+/**
+ * decompressBlockInto decompresses an LZ4 block into `dst` and returns the
+ * number of bytes written. Does not require knowing the uncompressed size
+ * ahead of time.
+ */
+export function decompressBlockInto(data, dst) {
+    return lz4.decompressBlock(data, dst, 0, data.length, 0);
+}

package/dist/ltx/page-index.d.ts

@@ -0,0 +1,26 @@
+/**
+ * LTX page index encoding/decoding.
+ *
+ * The page index is a sequence of varint triples `(pgno, offset, size)` in
+ * ascending page order, terminated by a zero varint, followed by an 8-byte
+ * big-endian size of the varint bytes (excluding the size field itself).
+ */
+import type { ByteReader } from './byte-reader.js';
+import type { PageIndex, TXID } from './types.js';
+/** Encodes a u64 as a little-endian LEB128 varint (same as Go's uvarint). */
+export declare function encodeUvarint(v: bigint): Uint8Array;
+/** Decodes a u64 varint from r, advancing its position. */
+export declare function readUvarint(r: ByteReader): bigint;
+/**
+ * Parses the page index section from r. observe, if non-null, receives every
+ * byte consumed (for checksumming); fn receives each record in file order.
+ */
+export declare function parsePageIndex(r: ByteReader, observe: ((buf: Uint8Array) => void) | null, fn: (pgno: number, offset: number, size: number) => void): void;
+/**
+ * DecodePageIndex decodes the page index from r, starting at the current
+ * position, into a Map keyed by page number.
+ */
+export declare function decodePageIndex(r: ByteReader, level: number, minTXID: TXID, maxTXID: TXID): PageIndex;
+/** Encodes a single page index record `(pgno, offset, size)` as a varint buffer. */
+export declare function encodePageIndexRecord(pgno: number, offset: number, size: number): Uint8Array;
+//# sourceMappingURL=page-index.d.ts.map

package/dist/ltx/page-index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page-index.d.ts","sourceRoot":"","sources":["../../src/ltx/page-index.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAKlD,6EAA6E;AAC7E,wBAAgB,aAAa,CAAC,CAAC,EAAE,MAAM,GAAG,UAAU,CAanD;AAED,2DAA2D;AAC3D,wBAAgB,WAAW,CAAC,CAAC,EAAE,UAAU,GAAG,MAAM,CAejD;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAC5B,CAAC,EAAE,UAAU,EACb,OAAO,EAAE,CAAC,CAAC,GAAG,EAAE,UAAU,KAAK,IAAI,CAAC,GAAG,IAAI,EAC3C,EAAE,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,KAAK,IAAI,GACvD,IAAI,CAqCN;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,CAAC,EAAE,UAAU,EACb,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,IAAI,EACb,OAAO,EAAE,IAAI,GACZ,SAAS,CAgBX;AAED,oFAAoF;AACpF,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,UAAU,CAe5F"}

package/dist/ltx/page-index.js

@@ -0,0 +1,118 @@
+/**
+ * LTX page index encoding/decoding.
+ *
+ * The page index is a sequence of varint triples `(pgno, offset, size)` in
+ * ascending page order, terminated by a zero varint, followed by an 8-byte
+ * big-endian size of the varint bytes (excluding the size field itself).
+ */
+import { readU64BE } from "./bytes.js";
+import { LTXFormatError } from "../errors.js";
+const MAX_INT64 = 0x7fffffffffffffffn;
+/** Encodes a u64 as a little-endian LEB128 varint (same as Go's uvarint). */
+export function encodeUvarint(v) {
+    const out = [];
+    for (;;) {
+        const b = Number(v & 0x7fn);
+        v >>= 7n;
+        if (v !== 0n) {
+            out.push(b | 0x80);
+        }
+        else {
+            out.push(b);
+            break;
+        }
+    }
+    return Uint8Array.from(out);
+}
+/** Decodes a u64 varint from r, advancing its position. */
+export function readUvarint(r) {
+    let x = 0n;
+    let s = 0n;
+    for (let i = 0; i < 10; i++) {
+        const b = r.readByte();
+        if (b < 0x80) {
+            if (i === 9 && b > 1) {
+                throw new LTXFormatError('uvarint overflows 64 bits');
+            }
+            return x | (BigInt(b) << s);
+        }
+        x |= BigInt(b & 0x7f) << s;
+        s += 7n;
+    }
+    throw new LTXFormatError('uvarint overflows 64 bits');
+}
+/**
+ * Parses the page index section from r. observe, if non-null, receives every
+ * byte consumed (for checksumming); fn receives each record in file order.
+ */
+export function parsePageIndex(r, observe, fn) {
+    let consumed = 0;
+    let prevPgno = 0;
+    for (;;) {
+        const start = r.pos;
+        const pgno = readUvarint(r);
+        if (pgno === 0n) {
+            // End marker.
+            observe?.(r.slice(start, r.pos));
+            consumed += r.pos - start;
+            break;
+        }
+        if (pgno > 0xffffffffn) {
+            throw new LTXFormatError(`page index pgno ${pgno} out of range`);
+        }
+        if (pgno <= BigInt(prevPgno)) {
+            throw new LTXFormatError(`page index out of order: ${pgno} after ${prevPgno}`);
+        }
+        const offset = readUvarint(r);
+        const size = readUvarint(r);
+        if (offset > MAX_INT64 || size > MAX_INT64) {
+            throw new LTXFormatError(`page index pgno ${pgno} offset/size out of range`);
+        }
+        observe?.(r.slice(start, r.pos));
+        consumed += r.pos - start;
+        prevPgno = Number(pgno);
+        fn(Number(pgno), Number(offset), Number(size));
+    }
+    // Read trailing size field.
+    const sizeBuf = r.readFull(8);
+    observe?.(sizeBuf);
+    const indexSize = readU64BE(sizeBuf, 0);
+    if (indexSize !== BigInt(consumed)) {
+        throw new LTXFormatError(`page index size mismatch: field=${indexSize} read=${consumed}`);
+    }
+}
+/**
+ * DecodePageIndex decodes the page index from r, starting at the current
+ * position, into a Map keyed by page number.
+ */
+export function decodePageIndex(r, level, minTXID, maxTXID) {
+    const index = new Map();
+    parsePageIndex(r, null, (pgno, offset, size) => {
+        index.set(pgno, {
+            level,
+            minTXID,
+            maxTXID,
+            offset,
+            size,
+        });
+    });
+    return index;
+}
+/** Encodes a single page index record `(pgno, offset, size)` as a varint buffer. */
+export function encodePageIndexRecord(pgno, offset, size) {
+    const parts = [
+        encodeUvarint(BigInt(pgno)),
+        encodeUvarint(BigInt(offset)),
+        encodeUvarint(BigInt(size)),
+    ];
+    let total = 0;
+    for (const p of parts)
+        total += p.length;
+    const out = new Uint8Array(total);
+    let o = 0;
+    for (const p of parts) {
+        out.set(p, o);
+        o += p.length;
+    }
+    return out;
+}

package/dist/ltx/page.d.ts

@@ -0,0 +1,27 @@
+/**
+ * LTX page header marshalling.
+ */
+import type { PageHeader } from './types.js';
+/** MarshalBinary encodes a PageHeader to its 6-byte representation. */
+export declare function marshalPageHeader(h: PageHeader): Uint8Array;
+/** UnmarshalBinary decodes a PageHeader from a byte slice. */
+export declare function unmarshalPageHeader(b: Uint8Array): PageHeader;
+/** Validate returns an error (thrown) if h is invalid. */
+export declare function validatePageHeader(h: PageHeader): void;
+/** The decoded result of a single LTX page frame. */
+export interface DecodedPage {
+    header: PageHeader;
+    data: Uint8Array;
+}
+/**
+ * DecodePageData decodes a single page frame from `b` (which must start at a
+ * page header). Matches Go's `ltx.DecodePageData`. For an empty (zero) page
+ * header, `data` is an empty buffer.
+ *
+ * Note: the uncompressed byte length is unknown here; LTX pages are at most
+ * `MaxPageSize`, so we decompress into a 64KiB buffer and slice to the bytes
+ * written. Callers that know the page size should use `DecodePage` in the
+ * decoder for the size-accurate path.
+ */
+export declare function decodePageData(b: Uint8Array): DecodedPage;
+//# sourceMappingURL=page.d.ts.map

package/dist/ltx/page.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"page.d.ts","sourceRoot":"","sources":["../../src/ltx/page.ts"],"names":[],"mappings":"AAAA;;GAEG;AAKH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAG7C,uEAAuE;AACvE,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,UAAU,GAAG,UAAU,CAK3D;AAED,8DAA8D;AAC9D,wBAAgB,mBAAmB,CAAC,CAAC,EAAE,UAAU,GAAG,UAAU,CAM7D;AAED,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAOtD;AAED,qDAAqD;AACrD,MAAM,WAAW,WAAW;IAC1B,MAAM,EAAE,UAAU,CAAC;IACnB,IAAI,EAAE,UAAU,CAAC;CAClB;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,CAAC,EAAE,UAAU,GAAG,WAAW,CA0BzD"}

package/dist/ltx/page.js

@@ -0,0 +1,66 @@
+/**
+ * LTX page header marshalling.
+ */
+import { PageHeaderSize, PageHeaderFlagSize, MaxPageSize } from "./constants.js";
+import { assertFull, readU16BE, readU32BE, writeU16BE, writeU32BE } from "./bytes.js";
+import { decompressBlockInto } from "./lz4.js";
+import { LTXFormatError } from "../errors.js";
+/** MarshalBinary encodes a PageHeader to its 6-byte representation. */
+export function marshalPageHeader(h) {
+    const b = new Uint8Array(PageHeaderSize);
+    writeU32BE(b, 0, h.pgno);
+    writeU16BE(b, 4, h.flags);
+    return b;
+}
+/** UnmarshalBinary decodes a PageHeader from a byte slice. */
+export function unmarshalPageHeader(b) {
+    assertFull(b, 0, PageHeaderSize);
+    return {
+        pgno: readU32BE(b, 0),
+        flags: readU16BE(b, 4),
+    };
+}
+/** Validate returns an error (thrown) if h is invalid. */
+export function validatePageHeader(h) {
+    if (h.pgno === 0) {
+        throw new LTXFormatError('page number required');
+    }
+    if ((h.flags & ~PageHeaderFlagSize) !== 0) {
+        throw new LTXFormatError(`invalid page header flags: 0x${h.flags.toString(16)}`);
+    }
+}
+/**
+ * DecodePageData decodes a single page frame from `b` (which must start at a
+ * page header). Matches Go's `ltx.DecodePageData`. For an empty (zero) page
+ * header, `data` is an empty buffer.
+ *
+ * Note: the uncompressed byte length is unknown here; LTX pages are at most
+ * `MaxPageSize`, so we decompress into a 64KiB buffer and slice to the bytes
+ * written. Callers that know the page size should use `DecodePage` in the
+ * decoder for the size-accurate path.
+ */
+export function decodePageData(b) {
+    const header = unmarshalPageHeader(b);
+    if (header.pgno === 0 && header.flags === 0) {
+        return { header, data: new Uint8Array(0) };
+    }
+    validatePageHeader(header);
+    if ((header.flags & PageHeaderFlagSize) !== 0) {
+        if (b.length < PageHeaderSize + 4) {
+            throw new LTXFormatError('buffer too small for size prefix');
+        }
+        const dataSize = readU32BE(b, PageHeaderSize);
+        const offset = PageHeaderSize + 4;
+        if (b.length < offset + dataSize) {
+            throw new LTXFormatError(`buffer too small for data: need ${offset + dataSize}, have ${b.length}`);
+        }
+        const compressed = b.subarray(offset, offset + dataSize);
+        const buf = new Uint8Array(MaxPageSize);
+        const n = decompressBlockInto(compressed, buf);
+        if (n <= 0) {
+            throw new LTXFormatError('decompress block failed');
+        }
+        return { header, data: buf.subarray(0, n) };
+    }
+    throw new LTXFormatError('legacy LZ4 frame format not supported');
+}

package/dist/ltx/sink.d.ts

@@ -0,0 +1,17 @@
+/**
+ * Output sink abstraction for the LTX encoder and other writers.
+ * A sink must accept ordered byte chunks; the encoder counts total bytes via
+ * its own `n` counter so sinks do not need to be seekable.
+ */
+export interface ByteSink {
+    write(bytes: Uint8Array): void;
+}
+/** In-memory sink that accumulates bytes into a single growable buffer. */
+export declare class BufferSink implements ByteSink {
+    private chunks;
+    private _length;
+    write(bytes: Uint8Array): void;
+    get length(): number;
+    toBuffer(): Uint8Array;
+}
+//# sourceMappingURL=sink.d.ts.map

package/dist/ltx/sink.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sink.d.ts","sourceRoot":"","sources":["../../src/ltx/sink.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI,CAAC;CAChC;AAED,2EAA2E;AAC3E,qBAAa,UAAW,YAAW,QAAQ;IACzC,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,OAAO,CAAK;IAEpB,KAAK,CAAC,KAAK,EAAE,UAAU,GAAG,IAAI;IAK9B,IAAI,MAAM,IAAI,MAAM,CAEnB;IAED,QAAQ,IAAI,UAAU;CASvB"}

package/dist/ltx/sink.js

@@ -0,0 +1,21 @@
+/** In-memory sink that accumulates bytes into a single growable buffer. */
+export class BufferSink {
+    chunks = [];
+    _length = 0;
+    write(bytes) {
+        this.chunks.push(Uint8Array.from(bytes));
+        this._length += bytes.length;
+    }
+    get length() {
+        return this._length;
+    }
+    toBuffer() {
+        const out = new Uint8Array(this._length);
+        let o = 0;
+        for (const c of this.chunks) {
+            out.set(c, o);
+            o += c.length;
+        }
+        return out;
+    }
+}

package/dist/ltx/stream.d.ts

@@ -0,0 +1,64 @@
+/**
+ * Streaming LTX reader.
+ *
+ * The in-memory {@link Decoder} is convenient but requires the whole LTX file in
+ * a single buffer. For large databases we read files from a streaming source (a
+ * `Readable`) and decode page-by-page, so peak memory stays bounded by one page
+ * buffer plus the small page index, never by the full database size.
+ *
+ * {@link StreamLtxReader} mirrors the synchronous `Decoder` in shape: read the
+ * header, then read each page into a reusable buffer, then `verify()` the page
+ * index + trailer (including the rolling checksum) once the end-of-pages marker
+ * is reached. It is used by streaming restore and compact.
+ */
+import type { Header, PageHeader, Trailer } from './types.js';
+/**
+ * Pulls bytes from an async iterable of chunks into an exact-bytes reader.
+ * Handles chunk boundaries that split logical records (page headers, LZ4
+ * blocks, varints) without buffering more than one chunk at a time.
+ */
+export declare class AsyncByteReader {
+    private it;
+    private buf;
+    private bufPos;
+    pos: number;
+    constructor(source: AsyncIterable<Uint8Array>);
+    /** Fills the internal buffer from the source. Returns false on EOF. */
+    private fill;
+    /** Reads exactly `n` bytes, advancing the position. Throws on short input. */
+    readFull(n: number): Promise<Uint8Array>;
+    /** Reads and returns every remaining byte in the source. */
+    readAllRemaining(): Promise<Uint8Array>;
+}
+/**
+ * Streams an LTX file from an async iterable of byte chunks, decoding the
+ * header and each page without holding the whole file in memory.
+ */
+export declare class StreamLtxReader {
+    private r;
+    private header;
+    private trailer;
+    private state;
+    private hash;
+    private pageSeq;
+    private chksum;
+    private pageN;
+    constructor(source: AsyncIterable<Uint8Array>);
+    /** Header returns the decoded header (after `readHeader`). */
+    Header(): Header;
+    /** Trailer returns the trailer (valid after `verify`). */
+    Trailer(): Trailer;
+    /** PageN returns the number of pages decoded. */
+    PageN(): number;
+    /** Reads and validates the LTX header frame. */
+    readHeader(): Promise<void>;
+    /**
+     * Reads the next page frame into `data`. Returns the page header, or null
+     * when the end-of-pages marker is reached.
+     */
+    readPage(data: Uint8Array): Promise<PageHeader | null>;
+    /** Reads the page index + trailer and verifies the file checksum. */
+    verify(): Promise<void>;
+    private hashPgo;
+}
+//# sourceMappingURL=stream.d.ts.map

package/dist/ltx/stream.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"stream.d.ts","sourceRoot":"","sources":["../../src/ltx/stream.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAoBH,OAAO,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,MAAM,YAAY,CAAC;AAa9D;;;;GAIG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,EAAE,CAA4B;IACtC,OAAO,CAAC,GAAG,CAAiC;IAC5C,OAAO,CAAC,MAAM,CAAK;IACnB,GAAG,SAAK;gBAEI,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC;IAI7C,uEAAuE;YACzD,IAAI;IAUlB,8EAA8E;IACxE,QAAQ,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,CAAC;IAgB9C,4DAA4D;IACtD,gBAAgB,IAAI,OAAO,CAAC,UAAU,CAAC;CAyB9C;AAED;;;GAGG;AACH,qBAAa,eAAe;IAC1B,OAAO,CAAC,CAAC,CAAkB;IAC3B,OAAO,CAAC,MAAM,CAAU;IACxB,OAAO,CAAC,OAAO,CAAwD;IACvE,OAAO,CAAC,KAAK,CAAe;IAC5B,OAAO,CAAC,IAAI,CAAe;IAC3B,OAAO,CAAC,OAAO,CAAe;IAC9B,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,KAAK,CAAK;gBAEN,MAAM,EAAE,aAAa,CAAC,UAAU,CAAC;IAI7C,8DAA8D;IAC9D,MAAM,IAAI,MAAM;IAIhB,0DAA0D;IAC1D,OAAO,IAAI,OAAO;IAIlB,iDAAiD;IACjD,KAAK,IAAI,MAAM;IAIf,gDAAgD;IAC1C,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAajC;;;OAGG;IACG,QAAQ,CAAC,IAAI,EAAE,UAAU,GAAG,OAAO,CAAC,UAAU,GAAG,IAAI,CAAC;IAmD5D,qEAAqE;IAC/D,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IA+D7B,OAAO,CAAC,OAAO;CAGhB"}