@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/src/leaser.ts

@@ -0,0 +1,86 @@
+/**
+ * Distributed leasing.
+ *
+ * When multiple replicas point at the same storage path, only one may be the
+ * active writer at a time. A `Leaser` acquires a lease on the replica's storage
+ * (a `lock.json` file / object); only the lease holder is the *primary* and may
+ * write. Other replicas become idle *followers* that poll until the lease is
+ * available.
+ *
+ * This mirrors litestream's `leaser.go`/`Lease` (a storage-agnostic interface)
+ * and `s3/leaser.go` (a conditional-write S3 implementation). Acquire uses
+ * `If-None-Match: *` when no lease exists and `If-Match: <etag>` (renew) or
+ * `If-Match: <etag>` (release) otherwise, so a stale writer can never clobber a
+ * live one.
+ */
+
+import { SqliteJsStreamError } from './errors.ts';
+
+/** A held lease. `etag` is the conditional-write token for renew/release. */
+export class Lease {
+  generation: number;
+  expiresAt: number; // ms since epoch
+  owner: string;
+  etag: string | null = null;
+
+  constructor(generation: number, expiresAt: number, owner: string, etag: string | null = null) {
+    this.generation = generation;
+    this.expiresAt = expiresAt;
+    this.owner = owner;
+    this.etag = etag;
+  }
+
+  /** IsExpired reports whether the lease has passed its expiration. */
+  isExpired(now = Date.now()): boolean {
+    return now > this.expiresAt;
+  }
+
+  /** TTL returns the remaining time before expiration, in ms. */
+  ttl(now = Date.now()): number {
+    return this.expiresAt - now;
+  }
+}
+
+/** The lease is already held by another owner. */
+export class LeaseExistsError extends SqliteJsStreamError {
+  owner: string;
+  expiresAt: number;
+
+  constructor(owner: string, expiresAt: number) {
+    super(
+      'ERR_LEASE_EXISTS',
+      owner !== ''
+        ? `lease already held by ${owner} until ${new Date(expiresAt).toISOString()}`
+        : `lease already held until ${new Date(expiresAt).toISOString()}`,
+    );
+    this.name = 'LeaseExistsError';
+    this.owner = owner;
+    this.expiresAt = expiresAt;
+  }
+}
+
+/** The caller no longer holds the lease (lost it or never acquired it). */
+export class ErrLeaseNotHeld extends SqliteJsStreamError {
+  constructor() {
+    super('ERR_LEASE_NOT_HELD', 'lease not held');
+    this.name = 'ErrLeaseNotHeld';
+  }
+}
+
+/**
+ * Leaser acquires, renews, and releases a lease on a storage path. The
+ * implementation must use conditional writes so concurrent acquirers cannot
+ * both win.
+ */
+export interface Leaser {
+  /** Acquires the lease, throwing {@link LeaseExistsError} if held elsewhere. */
+  acquireLease(): Promise<Lease>;
+  /** Renews the lease, throwing {@link ErrLeaseNotHeld} if we lost it. */
+  renewLease(lease: Lease): Promise<Lease>;
+  /** Releases the lease, throwing {@link ErrLeaseNotHeld} if we lost it. */
+  releaseLease(lease: Lease): Promise<void>;
+}
+
+export const DefaultLeaseTTL = 30_000; // 30s
+export const DefaultLeasePath = 'lock.json';
+export const DefaultLeaseRenewalBuffer = 5_000; // renew before it expires

package/src/ltx/byte-reader.ts

@@ -0,0 +1,46 @@
+/**
+ * A sequential byte reader over an in-memory buffer. Used by the LTX decoder
+ * to walk the page block, page index, and trailer while tracking position.
+ */
+
+import { assertFull } from './bytes.ts';
+import { LTXFormatError } from '../errors.ts';
+
+export class ByteReader {
+  private buf: Uint8Array;
+  pos: number;
+
+  constructor(buf: Uint8Array, start = 0) {
+    this.buf = buf;
+    this.pos = start;
+  }
+
+  get length(): number {
+    return this.buf.length;
+  }
+
+  get remaining(): number {
+    return this.buf.length - this.pos;
+  }
+
+  /** Reads one byte, advancing the position. Throws on EOF. */
+  readByte(): number {
+    if (this.pos >= this.buf.length) {
+      throw new LTXFormatError('unexpected end of input');
+    }
+    return this.buf[this.pos++];
+  }
+
+  /** Reads exactly `n` bytes, advancing the position. Throws on short input. */
+  readFull(n: number): Uint8Array {
+    assertFull(this.buf, this.pos, n);
+    const out = this.buf.subarray(this.pos, this.pos + n);
+    this.pos += n;
+    return out;
+  }
+
+  /** Returns a view of the bytes in [start, end) without moving the position. */
+  slice(start: number, end: number): Uint8Array {
+    return this.buf.subarray(start, end);
+  }
+}

package/src/ltx/bytes.ts

@@ -0,0 +1,87 @@
+/**
+ * Big-endian byte helpers for LTX marshalling.
+ */
+
+export function writeU8(buf: Uint8Array, off: number, v: number): void {
+  buf[off] = v & 0xff;
+}
+
+export function writeU16BE(buf: Uint8Array, off: number, v: number): void {
+  buf[off] = (v >>> 8) & 0xff;
+  buf[off + 1] = v & 0xff;
+}
+
+export function writeU32BE(buf: Uint8Array, off: number, v: number): void {
+  buf[off] = (v >>> 24) & 0xff;
+  buf[off + 1] = (v >>> 16) & 0xff;
+  buf[off + 2] = (v >>> 8) & 0xff;
+  buf[off + 3] = v & 0xff;
+}
+
+export function writeU64BE(buf: Uint8Array, off: number, v: bigint): void {
+  let n = v;
+  for (let i = 0; i < 8; i++) {
+    buf[off + (7 - i)] = Number(n & 0xffn);
+    n >>= 8n;
+  }
+}
+
+export function readU16BE(buf: Uint8Array, off: number): number {
+  return (buf[off] << 8) | buf[off + 1];
+}
+
+export function readU32BE(buf: Uint8Array, off: number): number {
+  return (
+    ((buf[off] | 0) << 24) >>> 0 |
+    (buf[off + 1] << 16) |
+    (buf[off + 2] << 8) |
+    buf[off + 3]
+  ) >>> 0;
+}
+
+export function readU32LE(buf: Uint8Array, off: number): number {
+  return (
+    buf[off] |
+    (buf[off + 1] << 8) |
+    (buf[off + 2] << 16) |
+    ((buf[off + 3] | 0) << 24)
+  ) >>> 0;
+}
+
+export function readU64BE(buf: Uint8Array, off: number): bigint {
+  let n = 0n;
+  for (let i = 0; i < 8; i++) {
+    n = (n << 8n) | BigInt(buf[off + i]);
+  }
+  return n;
+}
+
+/** Converts a number to a big-endian 4-byte array. */
+export function u32BE(v: number): Uint8Array {
+  const b = new Uint8Array(4);
+  writeU32BE(b, 0, v);
+  return b;
+}
+
+/** Converts a number to a big-endian 8-byte array. */
+export function u64BE(v: bigint): Uint8Array {
+  const b = new Uint8Array(8);
+  writeU64BE(b, 0, v);
+  return b;
+}
+
+/** Converts a number to a big-endian 2-byte array. */
+export function u16BE(v: number): Uint8Array {
+  const b = new Uint8Array(2);
+  writeU16BE(b, 0, v);
+  return b;
+}
+
+/** Throws unless exactly `n` bytes are available to read. */
+export function assertFull(buf: Uint8Array, off: number, n: number): void {
+  if (off + n > buf.length) {
+    throw new RangeError(
+      `short buffer: need ${n} bytes at offset ${off}, have ${buf.length - off}`,
+    );
+  }
+}

package/src/ltx/checksum.ts

@@ -0,0 +1,46 @@
+/**
+ * LTX checksum helpers: per-page checksums and the rolling DB checksum.
+ */
+
+import { ChecksumFlag, lockPgno } from './constants.ts';
+import { Crc64 } from './crc64.ts';
+import { u32BE } from './bytes.ts';
+
+/**
+ * ChecksumPage returns a CRC-64 checksum that combines the page number with
+ * the page data: `ChecksumFlag | CRC64-ISO(pgno_be4 ++ data)`.
+ */
+export function checksumPage(pgno: number, data: Uint8Array): bigint {
+  const h = new Crc64();
+  h.update(u32BE(pgno));
+  h.update(data);
+  return ChecksumFlag | h.sum64();
+}
+
+export function checksumPageWithHasher(h: Crc64, pgno: number, data: Uint8Array): bigint {
+  h.reset();
+  h.update(u32BE(pgno));
+  h.update(data);
+  return ChecksumFlag | h.sum64();
+}
+
+/**
+ * ChecksumReader computes the rolling DB checksum over a byte buffer split
+ * into pages of `pageSize`. The "lock page" is skipped entirely (its checksum
+ * contributes nothing), matching Go's `ChecksumReader`.
+ *
+ * For large databases this should later read page-by-page from a file rather
+ * than buffering the whole DB; provided as a buffer for Milestone 1.
+ */
+export function checksumReader(data: Uint8Array, pageSize: number): bigint {
+  const lockPg = lockPgno(pageSize);
+  let chksum = 0n;
+  let offset = 0;
+  for (let pgno = 1; offset < data.length; pgno++, offset += pageSize) {
+    const page = data.subarray(offset, offset + pageSize);
+    if (pgno !== lockPg) {
+      chksum = ChecksumFlag | (chksum ^ checksumPage(pgno, page));
+    }
+  }
+  return chksum;
+}

package/src/ltx/compaction-level.ts

@@ -0,0 +1,135 @@
+/**
+ * Multi-level compaction configuration.
+ *
+ * Mirrors litestream's `compaction_level.go`. Litestream stores raw LTX files
+ * at level 0 and progressively merges them into higher levels at increasing
+ * time intervals. Each level is compacted on its own wall-clock bucket; a level
+ * is compacted at most once per bucket so repeated writes are coalesced.
+ *
+ * Defaults (matching litestream):
+ *   - L0: raw LTX (no interval)
+ *   - L1: every 30s
+ *   - L2: every 5m
+ *   - L3: every 1h
+ */
+
+import { SnapshotLevel } from './constants.ts';
+import { InvalidArgumentError } from '../errors.ts';
+
+/** CompactionLevel describes a single compaction tier. */
+export interface CompactionLevel {
+  /** The numeric level. Level 0 is raw, never compacted. */
+  level: number;
+  /** Frequency (ms) that files are compacted from the previous level. 0 for level 0. */
+  intervalMs: number;
+}
+
+/** CompactionLevels is an ordered set of non-snapshot compaction levels. */
+export class CompactionLevels {
+  private levels: CompactionLevel[];
+
+  constructor(levels: CompactionLevel[]) {
+    this.levels = [...levels].sort((a, b) => a.level - b.level);
+    this.validate();
+  }
+
+  /** Returns the level configuration at `level`. */
+  level(level: number): CompactionLevel {
+    if (level === SnapshotLevel) {
+      throw new InvalidArgumentError('snapshot level has no compaction interval');
+    }
+    if (level < 0 || level > this.maxLevel()) {
+      throw new InvalidArgumentError(`level out of bounds: ${level}`);
+    }
+    return this.levels[level];
+  }
+
+  /** MaxLevel returns the highest non-snapshot compaction level. */
+  maxLevel(): number {
+    return this.levels.length - 1;
+  }
+
+  /** SnapshotLevel returns a pseudo level for full snapshots. */
+  snapshotLevel(): CompactionLevel {
+    return { level: SnapshotLevel, intervalMs: 0 };
+  }
+
+  /** PrevLevel returns the level that compacts into `level` (-1 if none). */
+  prevLevel(level: number): number {
+    if (level === SnapshotLevel) return this.maxLevel();
+    return level - 1;
+  }
+
+  /** NextLevel returns the level that `level` compacts into (-1 at the top). */
+  nextLevel(level: number): number {
+    if (level === SnapshotLevel) return -1;
+    if (level === this.maxLevel()) return SnapshotLevel;
+    return level + 1;
+  }
+
+  /** Validate checks the levels are ordered, bounded, and have sane intervals. */
+  validate(): void {
+    if (this.levels.length === 0) {
+      throw new InvalidArgumentError('at least one compaction level is required');
+    }
+    for (let i = 0; i < this.levels.length; i++) {
+      const lvl = this.levels[i];
+      if (lvl.level !== i) {
+        throw new InvalidArgumentError(
+          `compaction level number out of order: ${lvl.level}, expected ${i}`,
+        );
+      }
+      if (lvl.level > SnapshotLevel - 1) {
+        throw new InvalidArgumentError(`compaction level cannot exceed ${SnapshotLevel - 1}`);
+      }
+      if (lvl.level === 0 && lvl.intervalMs !== 0) {
+        throw new InvalidArgumentError('cannot set interval on compaction level zero');
+      }
+      if (lvl.level !== 0 && lvl.intervalMs <= 0) {
+        throw new InvalidArgumentError(`interval required for level ${lvl.level}`);
+      }
+    }
+  }
+
+  /** IsValidLevel returns true if `level` is a valid compaction level. */
+  isValidLevel(level: number): boolean {
+    if (level === SnapshotLevel) return true;
+    return level >= 0 && level < this.levels.length;
+  }
+
+  /** Returns a copy of the underlying levels. */
+  toArray(): CompactionLevel[] {
+    return this.levels.map((l) => ({ ...l }));
+  }
+}
+
+/** Canonical default compaction configuration (matches litestream). */
+export function defaultCompactionLevels(): CompactionLevels {
+  return new CompactionLevels([
+    { level: 0, intervalMs: 0 },
+    { level: 1, intervalMs: 30_000 },
+    { level: 2, intervalMs: 300_000 },
+    { level: 3, intervalMs: 3_600_000 },
+  ]);
+}
+
+/**
+ * PrevCompactionAt returns the start of the current compaction bucket for
+ * `intervalMs`. Compaction for a level only proceeds once per bucket, so
+ * repeated syncs across a window collapse into one compacted file.
+ */
+export function prevCompactionAt(now: number, intervalMs: number): number {
+  if (intervalMs <= 0) return now;
+  return Math.floor(now / intervalMs) * intervalMs;
+}
+
+/** NextCompactionAt returns the start of the next compaction bucket. */
+export function nextCompactionAt(now: number, intervalMs: number): number {
+  return prevCompactionAt(now, intervalMs) + intervalMs;
+}
+
+/** Interval (ms) constants matching litestream's defaults. */
+export const DefaultSnapshotIntervalMs = 24 * 60 * 60 * 1000; // 24h
+export const DefaultSnapshotRetentionMs = 24 * 60 * 60 * 1000; // 24h
+export const DefaultL0RetentionMs = 5 * 60 * 1000; // 5m
+export const DefaultL0RetentionCheckIntervalMs = 15 * 1000; // 15s

package/src/ltx/compactor.ts

@@ -0,0 +1,167 @@
+/**
+ * LTX Compactor merges multiple LTX files (readers) into a single stream,
+ * where later files override earlier ones for the same page number.
+ *
+ * Mirrors Go's `ltx.Compactor`. Used by restore (snapshot + incremental L0
+ * files) and by multi-level compaction.
+ */
+
+import type { Header, PageHeader, Trailer } from './types.ts';
+import { isZeroPageHeader, isContiguous } from './types.ts';
+import type { ByteSink } from './sink.ts';
+import { Encoder } from './encoder.ts';
+import { Decoder } from './decoder.ts';
+import { InvalidArgumentError, LTXFormatError, PageSizeMismatchError } from '../errors.ts';
+
+interface CompactorInput {
+  dec: Decoder;
+  hdr: PageHeader;
+  data: Uint8Array;
+}
+
+/** Compactor merges LTX readers into a single writer. */
+export class Compactor {
+  private enc: Encoder;
+  private inputs: CompactorInput[] = [];
+  private _headerFlags = 0;
+  private allowNonContiguousTXIDs = false;
+
+  constructor(sink: ByteSink, readers: Uint8Array[]) {
+    this.enc = new Encoder(sink);
+    for (const r of readers) {
+      const dec = new Decoder(r);
+      dec.setRetainPageIndex(false);
+      this.inputs.push({ dec, hdr: { pgno: 0, flags: 0 }, data: new Uint8Array(0) });
+    }
+  }
+
+  get headerFlags(): number {
+    return this._headerFlags;
+  }
+
+  set headerFlags(flags: number) {
+    this._headerFlags = flags;
+  }
+
+  /** AllowNonContiguousTXIDs disables contiguous-TXID validation between inputs. */
+  setAllowNonContiguousTXIDs(v: boolean): void {
+    this.allowNonContiguousTXIDs = v;
+  }
+
+  /** Header returns the output header (after Compact). */
+  Header(): Header {
+    return this.enc.Header();
+  }
+
+  /** Trailer returns the output trailer (after Compact). */
+  Trailer(): Trailer {
+    return this.enc.Trailer();
+  }
+
+  /** Compact merges all input readers into a single LTX stream. */
+  compact(): void {
+    if (this.inputs.length === 0) {
+      throw new InvalidArgumentError('at least one input reader required');
+    }
+
+    // Read headers from all inputs.
+    for (let i = 0; i < this.inputs.length; i++) {
+      this.inputs[i].dec.decodeHeader();
+    }
+
+    // Validate page sizes & contiguous TXIDs.
+    for (let i = 1; i < this.inputs.length; i++) {
+      const prevHdr = this.inputs[i - 1].dec.Header();
+      const hdr = this.inputs[i].dec.Header();
+      if (prevHdr.pageSize !== hdr.pageSize) {
+        throw new PageSizeMismatchError(
+          `input files have mismatched page sizes: ${prevHdr.pageSize} != ${hdr.pageSize}`,
+        );
+      }
+      if (
+        !this.allowNonContiguousTXIDs &&
+        !isContiguous(prevHdr.maxTXID, hdr.minTXID, hdr.maxTXID)
+      ) {
+        throw new LTXFormatError(
+          `non-contiguous transaction ids in input files: (${prevHdr.minTXID},${prevHdr.maxTXID}) -> (${hdr.minTXID},${hdr.maxTXID})`,
+        );
+      }
+    }
+
+    const minHdr = this.inputs[0].dec.Header();
+    const maxHdr = this.inputs[this.inputs.length - 1].dec.Header();
+
+    this.enc.encodeHeader({
+      version: 3,
+      flags: this.headerFlags,
+      pageSize: minHdr.pageSize,
+      commit: maxHdr.commit,
+      minTXID: minHdr.minTXID,
+      maxTXID: maxHdr.maxTXID,
+      timestamp: maxHdr.timestamp,
+      preApplyChecksum: minHdr.preApplyChecksum,
+      walOffset: 0n,
+      walSize: 0n,
+      walSalt1: 0,
+      walSalt2: 0,
+      nodeID: 0n,
+    });
+
+    this.writePageBlock();
+
+    for (const input of this.inputs) {
+      input.dec.close();
+    }
+
+    const last = this.inputs[this.inputs.length - 1];
+    this.enc.setPostApplyChecksum(last.dec.Trailer().postApplyChecksum);
+    this.enc.close();
+  }
+
+  private writePageBlock(): void {
+    const pageSize = this.enc.Header().pageSize;
+    for (const input of this.inputs) {
+      input.data = new Uint8Array(pageSize);
+    }
+    for (;;) {
+      const pgno = this.fillPageBuffers();
+      if (pgno === 0) break;
+      this.writePageBuffer(pgno);
+    }
+  }
+
+  /** Fills input buffers and returns the smallest available page number. */
+  private fillPageBuffers(): number {
+    let pgno = 0;
+    for (const input of this.inputs) {
+      if (isZeroPageHeader(input.hdr)) {
+        const hdr = input.dec.decodePage(input.data);
+        if (hdr === null) continue; // end of page block for this input
+        input.hdr = hdr;
+      }
+      if (pgno === 0 || input.hdr.pgno < pgno) {
+        pgno = input.hdr.pgno;
+      }
+    }
+    return pgno;
+  }
+
+  /** Writes the buffer with a matching pgno from the latest input. */
+  private writePageBuffer(pgno: number): void {
+    const commit = this.enc.Header().commit;
+    let pageWritten = false;
+    for (let i = this.inputs.length - 1; i >= 0; i--) {
+      const input = this.inputs[i];
+      if (input.hdr.pgno !== pgno) continue;
+
+      const hdr = input.hdr;
+      const data = input.data;
+      input.hdr = { pgno: 0, flags: 0 };
+
+      if (pageWritten) continue;
+      if (pgno > commit) continue;
+      pageWritten = true;
+      this.enc.encodePage(hdr, data);
+    }
+  }
+}

package/src/ltx/constants.ts

@@ -0,0 +1,71 @@
+/**
+ * LTX format constants.
+ *
+ * Ground truth: superfly/ltx v0.5.2 (vendored by litestream). The byte layout
+ * here must match Go's `ltx` package exactly so files interoperate.
+ */
+
+/** Magic is the first 4 bytes of every LTX file. */
+export const Magic = 'LTX1';
+
+/** Version is the current version of the LTX file format. */
+export const Version = 3;
+
+/** Size constants. */
+export const HeaderSize = 100;
+export const PageHeaderSize = 6;
+export const TrailerSize = 16;
+
+/** Checksum size & positions. */
+export const ChecksumSize = 8;
+export const TrailerChecksumOffset = TrailerSize - ChecksumSize;
+
+/** ChecksumFlag ensures a checksum is non-zero. */
+export const ChecksumFlag = 1n << 63n;
+
+/** Entry byte offset of the PENDING_BYTE within a database. */
+export const PENDING_BYTE = 0x40000000;
+
+/** Header flags. */
+export const HeaderFlagMask = 0x00000002;
+export const HeaderFlagNoChecksum = 0x00000002;
+
+/** Page header flags. */
+/** PageHeaderFlagSize indicates a 4-byte size field follows the page header. */
+export const PageHeaderFlagSize = 0x0001;
+
+/** MaxPageSize is the maximum allowed SQLite page size. */
+export const MaxPageSize = 65536;
+
+/** SnapshotLevel is the LTX level reserved for full-database snapshots. */
+export const SnapshotLevel = 9;
+
+/** Error messages / codes. */
+export const ErrInvalidFile = 'invalid LTX file';
+export const ErrDecoderClosed = 'ltx decoder closed';
+export const ErrEncoderClosed = 'ltx encoder closed';
+export const ErrEncoderAborted = 'ltx encoder aborted';
+export const ErrNoChecksum = 'no file checksum';
+export const ErrInvalidChecksumFormat = 'invalid file checksum format';
+export const ErrChecksumMismatch = 'file checksum mismatch';
+
+/**
+ * LockPgno returns the page number at which the PENDING_BYTE exists.
+ * This page is never part of the LTX payload.
+ */
+export function lockPgno(pageSize: number): number {
+  return Math.floor(PENDING_BYTE / pageSize) + 1;
+}
+
+/** IsValidPageSize returns true if sz is between 512 and 64K and a power of two. */
+export function isValidPageSize(sz: number): boolean {
+  for (let i = 512; i <= MaxPageSize; i *= 2) {
+    if (sz === i) return true;
+  }
+  return false;
+}
+
+/** IsValidHeaderFlags returns true unless flags outside the valid mask are set. */
+export function isValidHeaderFlags(flags: number): boolean {
+  return flags === (flags & HeaderFlagMask);
+}