@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/src/ltx/crc64.ts

@@ -0,0 +1,65 @@
+/**
+ * CRC-64 ISO implementation, byte-compatible with Go's `hash/crc64` using
+ * `crc64.MakeTable(crc64.ISO)`. This is what superfly/ltx uses for all LTX
+ * checksums (page checksums, rolling DB checksum, file checksum).
+ *
+ * NOTE: LTX uses CRC-64 ISO, NOT ECMA. The litestream docs are misleading;
+ * the actual code (`crc64.New(crc64.MakeTable(crc64.ISO))`) uses ISO.
+ *
+ * Algorithm matches Go's reflected `crc64.Update`:
+ *   - init / xorout = all ones (0xFFFFFFFFFFFFFFFF)
+ *   - polynomial register value = 0xD800000000000000
+ *   - table built by right-shifting (reflected), update as
+ *     `tab[byte(crc) ^ v] ^ (crc >> 8)`
+ */
+
+const ISO_POLY = 0xd800000000000000n;
+const MASK64 = 0xffffffffffffffffn;
+
+/** Builds the 256-entry CRC table for a reflected polynomial. */
+function makeTable(poly: bigint): bigint[] {
+  const table: bigint[] = new Array(256);
+  for (let i = 0; i < 256; i++) {
+    let crc = BigInt(i);
+    for (let j = 0; j < 8; j++) {
+      crc = (crc & 1n) !== 0n ? (crc >> 1n) ^ poly : crc >> 1n;
+    }
+    table[i] = crc;
+  }
+  return table;
+}
+
+const ISO_TABLE = makeTable(ISO_POLY);
+
+/**
+ * Crc64 is a streaming CRC-64 hasher, equivalent to Go's `hash/crc64`
+ * (ISO table). Call `update()` with successive byte chunks and `sum64()` to
+ * obtain the running checksum. `reset()` re-initializes the hasher.
+ */
+export class Crc64 {
+  private crc: bigint = MASK64;
+
+  update(buf: Uint8Array): void {
+    let crc = this.crc;
+    for (let i = 0; i < buf.length; i++) {
+      crc = ISO_TABLE[Number((crc ^ BigInt(buf[i])) & 0xffn)] ^ (crc >> 8n);
+    }
+    this.crc = crc;
+  }
+
+  reset(): void {
+    this.crc = MASK64;
+  }
+
+  /** Returns the digest of all data written so far. */
+  sum64(): bigint {
+    return this.crc ^ MASK64;
+  }
+}
+
+/** Convenience: computes the CRC-64 ISO of a single buffer. */
+export function crc64ISO(buf: Uint8Array): bigint {
+  const h = new Crc64();
+  h.update(buf);
+  return h.sum64();
+}

package/src/ltx/decoder.ts

@@ -0,0 +1,282 @@
+/**
+ * LTX decoder.
+ *
+ * Mirrors Go's `ltx.Decoder`. Reads header, page frames, page index, and
+ * trailer from an in-memory buffer, verifying the file checksum and (for
+ * checksum-tracking snapshots) the post-apply rolling checksum.
+ */
+
+import {
+  ChecksumFlag,
+  HeaderSize,
+  PageHeaderFlagSize,
+  PageHeaderSize,
+  TrailerChecksumOffset,
+  TrailerSize,
+  lockPgno,
+} from './constants.ts';
+import { Crc64 } from './crc64.ts';
+import { u32BE } from './bytes.ts';
+import { ByteReader } from './byte-reader.ts';
+import type { ByteSink } from './sink.ts';
+import type { Header, PageHeader, PageIndex, Trailer } from './types.ts';
+import { isSnapshot } from './types.ts';
+import { unmarshalHeader, validateHeader } from './header.ts';
+import { unmarshalPageHeader } from './page.ts';
+import { unmarshalTrailer } from './trailer.ts';
+import { parsePageIndex } from './page-index.ts';
+import { decompressBlock } from './lz4.ts';
+import { checksumPage } from './checksum.ts';
+import { LTXFormatError } from '../errors.ts';
+
+const stateHeader = 'header';
+const statePage = 'page';
+const stateClose = 'close';
+const stateClosed = 'closed';
+
+function noChecksum(h: Header): boolean {
+  return (h.flags & 0x00000002) !== 0;
+}
+
+/** Decoder reads an LTX file from an in-memory buffer. */
+export class Decoder {
+  private r: ByteReader;
+  private header!: Header;
+  private trailer: Trailer = { postApplyChecksum: 0n, fileChecksum: 0n };
+  private pageIndex: PageIndex | null = null;
+  private state = stateHeader;
+  private hash = new Crc64();
+  private pageSeq = new Crc64();
+  private chksum = 0n;
+  private pageN = 0;
+  private retainPageIndex: boolean;
+
+  constructor(buf: Uint8Array) {
+    this.r = new ByteReader(buf);
+    this.retainPageIndex = true;
+  }
+
+  /** SetRetainPageIndex controls whether Close materializes the page-index map. */
+  setRetainPageIndex(retain: boolean): void {
+    this.retainPageIndex = retain;
+  }
+
+  /** N returns the number of source bytes read. */
+  N(): number {
+    return this.r.pos;
+  }
+
+  /** PageN returns the number of pages decoded. */
+  PageN(): number {
+    return this.pageN;
+  }
+
+  /** Header returns a copy of the header. */
+  Header(): Header {
+    return this.header;
+  }
+
+  /** Trailer returns the trailer. File checksum valid after Close(). */
+  Trailer(): Trailer {
+    return this.trailer;
+  }
+
+  /** PageIndex returns the page index (after Close), or null if not retained. */
+  PageIndex(): PageIndex | null {
+    return this.pageIndex;
+  }
+
+  /** PostApplyPos returns the replication position after applying this file. */
+  PostApplyPos() {
+    return {
+      txID: this.header.maxTXID,
+      postApplyChecksum: this.trailer.postApplyChecksum,
+    };
+  }
+
+  /** DecodeHeader reads and validates the header frame. */
+  decodeHeader(): void {
+    const b = this.r.readFull(HeaderSize);
+    this.header = unmarshalHeader(b);
+    this.hash.update(b);
+    this.state = statePage;
+    validateHeader(this.header);
+    if (!noChecksum(this.header)) {
+      this.chksum = ChecksumFlag;
+    }
+  }
+
+  /**
+   * DecodePage reads the next page frame into `data`. Returns the page header,
+   * or null when the end-of-pages marker is reached.
+   */
+  decodePage(data: Uint8Array): PageHeader | null {
+    if (this.state === stateClosed) throw new LTXFormatError('ltx decoder closed');
+    if (this.state === stateClose) return null;
+    if (this.state !== statePage) {
+      throw new LTXFormatError(`cannot read page header, expected ${this.state}`);
+    }
+    if (data.length !== this.header.pageSize) {
+      throw new LTXFormatError(`invalid page buffer size: ${data.length}, expecting ${this.header.pageSize}`);
+    }
+
+    const b = this.r.readFull(PageHeaderSize);
+    const hdr = unmarshalPageHeader(b);
+    this.hash.update(b);
+
+    // An empty page header marks the end of the page block.
+    if (hdr.pgno === 0 && hdr.flags === 0) {
+      this.state = stateClose;
+      return null;
+    }
+
+    if (hdr.pgno > this.header.commit) {
+      throw new LTXFormatError(`page number ${hdr.pgno} out-of-bounds for commit size ${this.header.commit}`);
+    }
+
+    if (hdr.flags & PageHeaderFlagSize) {
+      // New block format: read size prefix, then LZ4 block data.
+      const sizeBuf = this.r.readFull(4);
+      this.hash.update(sizeBuf);
+      const dataSize = (sizeBuf[0] << 24 | sizeBuf[1] << 16 | sizeBuf[2] << 8 | sizeBuf[3]) >>> 0;
+      const compressed = this.r.readFull(dataSize);
+      const decompressed = decompressBlock(compressed, this.header.pageSize);
+      data.set(decompressed);
+    } else {
+      throw new LTXFormatError('legacy LZ4 frame format not supported');
+    }
+
+    this.hash.update(data);
+    this.hashPgo(this.pageSeq, hdr.pgno);
+    this.pageN++;
+
+    // Track the rolling DB checksum for checksum-tracking snapshots.
+    if (isSnapshot(this.header) && !noChecksum(this.header)) {
+      if (hdr.pgno !== lockPgno(this.header.pageSize)) {
+        this.chksum = ChecksumFlag | (this.chksum ^ checksumPage(hdr.pgno, data));
+      }
+    }
+
+    return hdr;
+  }
+
+  /** Verify reads the entire file, discarding page data (for validation). */
+  verify(): void {
+    this.decodeHeader();
+    const data = new Uint8Array(this.header.pageSize);
+    for (;;) {
+      if (this.decodePage(data) === null) break;
+    }
+    this.close();
+  }
+
+  /** DecodeDatabaseTo writes the snapshot as a SQLite database to `sink`. */
+  decodeDatabaseTo(sink: ByteSink): void {
+    this.decodeHeader();
+    const hdr = this.header;
+    if (!isSnapshot(hdr)) {
+      throw new LTXFormatError('cannot decode non-snapshot LTX file to SQLite database');
+    }
+    const lockPg = lockPgno(hdr.pageSize);
+    const data = new Uint8Array(hdr.pageSize);
+    for (let pgno = 1; pgno <= hdr.commit; pgno++) {
+      if (pgno === lockPg) {
+        sink.write(new Uint8Array(hdr.pageSize));
+        continue;
+      }
+      const ph = this.decodePage(data);
+      if (ph === null) {
+        throw new LTXFormatError(`unexpected end of pages at pgno ${pgno}`);
+      }
+      if (ph.pgno !== pgno) {
+        throw new LTXFormatError(`unexpected pgno while decoding page: read ${ph.pgno}, expected ${pgno}`);
+      }
+      sink.write(data);
+    }
+
+    // Expect one final read to be the end-of-pages marker.
+    if (this.decodePage(data) !== null) {
+      throw new LTXFormatError('unexpected page after end of database');
+    }
+    this.close();
+  }
+
+  /** Close verifies the page index and trailer, then checks the file checksum. */
+  close(): void {
+    if (this.state === stateClosed) return;
+    if (this.state !== stateClose) {
+      throw new LTXFormatError(`cannot close, expected ${this.state}`);
+    }
+
+    const index: PageIndex = new Map();
+    let n = 0;
+    let nextOffset = HeaderSize;
+    const seq = new Crc64();
+    const self = this;
+
+    parsePageIndex(
+      this.r,
+      (buf) => self.hash.update(buf),
+      (pgno, offset, size) => {
+        if (pgno > self.header.commit) {
+          throw new LTXFormatError(`page index pgno ${pgno} exceeds commit ${self.header.commit}`);
+        }
+        if (offset < nextOffset) {
+          throw new LTXFormatError(
+            `page index pgno ${pgno} offset ${offset} overlaps previous frame ending at ${nextOffset}`,
+          );
+        }
+        if (size <= PageHeaderSize) {
+          throw new LTXFormatError(`page index pgno ${pgno} has invalid frame size ${size}`);
+        }
+        nextOffset = offset + size;
+        n++;
+        self.hashPgo(seq, pgno);
+        if (index) {
+          index.set(pgno, {
+            level: 0,
+            minTXID: self.header.minTXID,
+            maxTXID: self.header.maxTXID,
+            offset,
+            size,
+          });
+        }
+      },
+    );
+
+    if (n !== this.pageN) {
+      throw new LTXFormatError(`page index has ${n} entries but ${this.pageN} pages were decoded`);
+    }
+    if (seq.sum64() !== this.pageSeq.sum64()) {
+      throw new LTXFormatError('page index does not match the decoded page numbers');
+    }
+    this.pageIndex = this.retainPageIndex ? index : null;
+
+    // Read the trailer; everything except the file checksum field is hashed.
+    const t = this.r.readFull(TrailerSize);
+    this.hash.update(t.subarray(0, TrailerChecksumOffset));
+    this.trailer = unmarshalTrailer(t);
+    if (this.r.remaining > 0) {
+      throw new LTXFormatError('unexpected data after trailer');
+    }
+
+    const computed = ChecksumFlag | this.hash.sum64();
+    if (computed !== this.trailer.fileChecksum) {
+      throw new LTXFormatError('file checksum mismatch');
+    }
+
+    if (isSnapshot(this.header) && !noChecksum(this.header)) {
+      if (this.trailer.postApplyChecksum !== this.chksum) {
+        throw new LTXFormatError(
+          `post-apply checksum in trailer (${this.trailer.postApplyChecksum.toString(16)}) does not match calculated checksum (${this.chksum.toString(16)})`,
+        );
+      }
+    }
+
+    this.state = stateClosed;
+  }
+
+  private hashPgo(h: Crc64, pgno: number): void {
+    h.update(u32BE(pgno));
+  }
+}

package/src/ltx/encoder.ts

@@ -0,0 +1,200 @@
+/**
+ * LTX encoder.
+ *
+ * Mirrors Go's `ltx.Encoder`. The encoder streams header, page frames, page
+ * index, and trailer to a `ByteSink`, maintaining a running CRC-64 ISO file
+ * checksum over all logical content (header, page headers, compressed-size
+ * prefixes, *decompressed* page data, page index bytes, and the trailer's
+ * post-apply checksum) plus a byte counter for page-index offsets.
+ */
+
+import {
+  ChecksumFlag,
+  PageHeaderFlagSize,
+  PageHeaderSize,
+  TrailerChecksumOffset,
+  TrailerSize,
+  lockPgno,
+} from './constants.ts';
+import { Crc64 } from './crc64.ts';
+import { u32BE, u64BE } from './bytes.ts';
+import type { ByteSink } from './sink.ts';
+import type { Header, PageHeader, Trailer } from './types.ts';
+import { marshalHeader, validateHeader } from './header.ts';
+import { marshalPageHeader, validatePageHeader } from './page.ts';
+import { marshalTrailer, validateTrailer } from './trailer.ts';
+import { encodePageIndexRecord, encodeUvarint } from './page-index.ts';
+import { compressBlock } from './lz4.ts';
+import { LTXFormatError } from '../errors.ts';
+
+const stateHeader = 'header';
+const statePage = 'page';
+const stateClosed = 'closed';
+
+interface IndexEntry {
+  pgno: number;
+  offset: number;
+  size: number;
+}
+
+/** Encoder writes an LTX file to a ByteSink. */
+export class Encoder {
+  private sink: ByteSink;
+  private state: string = stateHeader;
+  private header!: Header;
+  private trailer: Trailer = { postApplyChecksum: 0n, fileChecksum: 0n };
+  private hash = new Crc64();
+  private index: IndexEntry[] = [];
+  private n = 0;
+  private prevPgno = 0;
+  private pagesWritten = 0;
+
+  constructor(sink: ByteSink) {
+    this.sink = sink;
+  }
+
+  /** Returns the number of bytes written to the sink so far. */
+  N(): number {
+    return this.n;
+  }
+
+  /** Returns a shallow copy of the header. */
+  Header(): Header {
+    return this.header;
+  }
+
+  /** Returns the trailer (file checksum valid after Close()). */
+  Trailer(): Trailer {
+    return this.trailer;
+  }
+
+  /** Returns the replication position after applying this file. */
+  PostApplyPos() {
+    return {
+      txID: this.header.maxTXID,
+      postApplyChecksum: this.trailer.postApplyChecksum,
+    };
+  }
+
+  /** SetPostApplyChecksum sets the database checksum to record in the trailer. */
+  setPostApplyChecksum(chk: bigint): void {
+    this.trailer.postApplyChecksum = chk;
+  }
+
+  /** EncodeHeader writes the header frame and initializes the checksum. */
+  encodeHeader(h: Header): void {
+    if (this.state === stateClosed) throw new LTXFormatError('ltx encoder closed');
+    if (this.state !== stateHeader) {
+      throw new LTXFormatError(`cannot encode header frame, expected ${this.state}`);
+    }
+    validateHeader(h);
+    this.header = h;
+    this.hash = new Crc64();
+    this.write(marshalHeader(h));
+    this.state = statePage;
+  }
+
+  /**
+   * EncodePage writes a single page frame. `data` must be exactly the page
+   * size and `hdr.pgno` must be in ascending order and not the lock page.
+   */
+  encodePage(hdr: PageHeader, data: Uint8Array): void {
+    if (this.state === stateClosed) throw new LTXFormatError('ltx encoder closed');
+    if (this.state !== statePage) {
+      throw new LTXFormatError(`cannot encode page header, expected ${this.state}`);
+    }
+    if (hdr.pgno > this.header.commit) {
+      throw new LTXFormatError(
+        `page number ${hdr.pgno} out-of-bounds for commit size ${this.header.commit}`,
+      );
+    }
+    validatePageHeader(hdr);
+    if (data.length !== this.header.pageSize) {
+      throw new LTXFormatError(
+        `invalid page buffer size: ${data.length}, expecting ${this.header.pageSize}`,
+      );
+    }
+
+    const lockPg = lockPgno(this.header.pageSize);
+    if (hdr.pgno === lockPg) {
+      throw new LTXFormatError(`cannot encode lock page: pgno=${hdr.pgno}`);
+    }
+
+    // Enforce page ordering.
+    if (this.header.minTXID === 1n) {
+      // Snapshot files must include all pages up to commit, in order.
+      if (this.prevPgno === 0 && hdr.pgno !== 1) {
+        throw new LTXFormatError('snapshot transaction file must start with page number 1');
+      }
+      if (this.prevPgno === lockPg - 1) {
+        if (hdr.pgno !== this.prevPgno + 2) {
+          throw new LTXFormatError(
+            `nonsequential page numbers in snapshot transaction (skip lock page): ${this.prevPgno},${hdr.pgno}`,
+          );
+        }
+      } else if (this.prevPgno !== 0 && hdr.pgno !== this.prevPgno + 1) {
+        throw new LTXFormatError(
+          `nonsequential page numbers in snapshot transaction: ${this.prevPgno},${hdr.pgno}`,
+        );
+      }
+    } else {
+      if (this.prevPgno >= hdr.pgno) {
+        throw new LTXFormatError(`out-of-order page numbers: ${this.prevPgno},${hdr.pgno}`);
+      }
+    }
+
+    const offset = this.n;
+    const compressed = compressBlock(data);
+
+    const fhdr: PageHeader = { pgno: hdr.pgno, flags: hdr.flags | PageHeaderFlagSize };
+    this.write(marshalPageHeader(fhdr));
+    this.write(u32BE(compressed.length));
+    this.sink.write(compressed);
+    this.hash.update(data); // hash the raw (uncompressed) page data
+    this.n += compressed.length;
+
+    this.pagesWritten++;
+    this.prevPgno = hdr.pgno;
+    this.index.push({ pgno: hdr.pgno, offset, size: this.n - offset });
+  }
+
+  /** Close writes the end-of-pages marker, page index, and trailer. */
+  close(): void {
+    if (this.state === stateClosed) return;
+    if (this.state !== statePage) {
+      throw new LTXFormatError(`cannot close, expected ${this.state}`);
+    }
+
+    // Empty page header terminates the page block.
+    const zero = new Uint8Array(PageHeaderSize);
+    this.write(zero);
+
+    // Write the page index.
+    const start = this.n;
+    for (const e of this.index) {
+      this.write(encodePageIndexRecord(e.pgno, e.offset, e.size));
+    }
+    this.write(encodeUvarint(0n)); // end marker
+    this.write(u64BE(BigInt(this.n - start))); // size of index varint bytes
+
+    // Marshal trailer (post-apply checksum set, file checksum pending).
+    const t = marshalTrailer(this.trailer);
+    this.hash.update(t.subarray(0, TrailerChecksumOffset));
+    this.n += TrailerChecksumOffset;
+    this.trailer.fileChecksum = ChecksumFlag | this.hash.sum64();
+    validateTrailer(this.trailer, this.header);
+
+    // Remarshal with the file checksum and write to the sink (not hashed).
+    this.sink.write(marshalTrailer(this.trailer));
+    this.n += TrailerSize - TrailerChecksumOffset;
+
+    this.state = stateClosed;
+  }
+
+  /** Writes to the sink, feeding the checksum and byte counter. */
+  private write(b: Uint8Array): void {
+    this.sink.write(b);
+    this.hash.update(b);
+    this.n += b.length;
+  }
+}

package/src/ltx/file-info.ts

@@ -0,0 +1,81 @@
+/**
+ * LTX file metadata and iteration over a set of LTX files.
+ */
+
+import type { FileInfo, TXID } from './types.ts';
+import { isContiguous } from './types.ts';
+
+/** Sorts FileInfo by (level, minTXID, maxTXID), in place. */
+export function sortFileInfo(a: FileInfo[]): FileInfo[] {
+  a.sort((x, y) => {
+    if (x.level !== y.level) return x.level - y.level;
+    if (x.minTXID !== y.minTXID) return x.minTXID < y.minTXID ? -1 : 1;
+    return x.maxTXID < y.maxTXID ? -1 : x.maxTXID > y.maxTXID ? 1 : 0;
+  });
+  return a;
+}
+
+/** MaxTXID returns the largest maxTXID in a slice, or 0n if empty. */
+export function maxTXID(a: readonly FileInfo[]): TXID {
+  if (a.length === 0) return 0n;
+  let m = a[0].maxTXID;
+  for (const info of a) {
+    if (info.maxTXID > m) m = info.maxTXID;
+  }
+  return m;
+}
+
+/** SliceFileIterator iterates over a sorted slice of LTX file metadata. */
+export class FileInfoSliceIterator {
+  private a: FileInfo[];
+  private init = false;
+  private current: FileInfo | null = null;
+
+  constructor(a: FileInfo[]) {
+    this.a = sortFileInfo([...a]);
+  }
+
+  next(): boolean {
+    if (!this.init) {
+      this.init = true;
+      if (this.a.length === 0) {
+        this.current = null;
+        return false;
+      }
+      this.current = this.a[0];
+      return true;
+    }
+    this.a = this.a.slice(1);
+    if (this.a.length === 0) {
+      this.current = null;
+      return false;
+    }
+    this.current = this.a[0];
+    return true;
+  }
+
+  err(): Error | null {
+    return null;
+  }
+
+  item(): FileInfo | null {
+    return this.current;
+  }
+
+  close(): void {
+    /* nothing to release */
+  }
+}
+
+/** Returns all LTX files from an iterator as an array. */
+export function sliceFileIterator(itr: FileInfoSliceIterator): FileInfo[] {
+  const out: FileInfo[] = [];
+  while (itr.next()) {
+    const item = itr.item();
+    if (item) out.push(item);
+  }
+  itr.close();
+  return out;
+}
+
+export { isContiguous };

package/src/ltx/filename.ts

@@ -0,0 +1,27 @@
+/**
+ * LTX filename parsing/formatting.
+ *
+ * Names are `{minTXID:016x}-{maxTXID:016x}.ltx`.
+ */
+
+import { formatTXID } from './types.ts';
+import { LTXFormatError } from '../errors.ts';
+
+const FILENAME_RE = /^([0-9a-f]{16})-([0-9a-f]{16})\.ltx$/;
+
+/** FormatFilename returns an LTX filename representing a TXID range. */
+export function formatFilename(minTXID: bigint, maxTXID: bigint): string {
+  return `${formatTXID(minTXID)}-${formatTXID(maxTXID)}.ltx`;
+}
+
+/** ParseFilename parses a transaction range from an LTX filename. */
+export function parseFilename(name: string): { minTXID: bigint; maxTXID: bigint } {
+  const m = FILENAME_RE.exec(name);
+  if (m === null) {
+    throw new LTXFormatError(`invalid ltx filename: ${name}`);
+  }
+  return {
+    minTXID: BigInt(`0x${m[1]}`),
+    maxTXID: BigInt(`0x${m[2]}`),
+  };
+}