@platformatic/sqlite-js-stream 0.0.0-stage → 0.1.1

package/src/store.ts

@@ -0,0 +1,71 @@
+/**
+ * Store manages one or more Replicas (a litestream instance).
+ *
+ * MVP: a thin orchestration layer that opens replicas for database paths with
+ * their replica clients, syncs/snapshots them, and closes. Monitor loops,
+ * scheduling, and retention are future work.
+ */
+
+import type { Replica, RetentionOptions, ReplicaRunOptions } from './replica.ts';
+import { Replica as ReplicaClass } from './replica.ts';
+import type { ReplicaClient } from './replica-client.ts';
+
+/** Store coordinates multiple replicated databases. */
+export class Store {
+  private items: Replica[] = [];
+
+  /** OpenDB opens a replica for a database path and wires it to a client. */
+  async openDB(path: string, client: ReplicaClient): Promise<Replica> {
+    const replica = new ReplicaClass(path, client);
+    this.items.push(replica);
+    return replica;
+  }
+
+  /** Replicas returns all open replicas. */
+  replicas(): Replica[] {
+    return [...this.items];
+  }
+
+  /** SyncAll syncs every database (local writes then replica upload). */
+  async syncAll(): Promise<void> {
+    for (const replica of this.items) {
+      await replica.sync();
+    }
+  }
+
+  /** SnapshotAll writes & uploads level-9 snapshots for every database. */
+  async snapshotAll(): Promise<void> {
+    for (const replica of this.items) {
+      await replica.snapshot();
+    }
+  }
+
+  /** CompactAll compacts pending LTX files for every database. */
+  async compactAll(): Promise<void> {
+    for (const replica of this.items) {
+      await replica.compact();
+    }
+  }
+
+  /** EnforceRetentionAll prunes old snapshots/files for every database. */
+  async enforceRetentionAll(options: RetentionOptions = {}): Promise<void> {
+    for (const replica of this.items) {
+      await replica.enforceRetention(options);
+    }
+  }
+
+  /**
+   * Run starts each replica's continuous sync/compaction/retention loop and
+   * resolves once `options.signal` is aborted. Pass per-replica intervals and a
+   * retention policy via {@link ReplicaRunOptions}.
+   */
+  async run(options: ReplicaRunOptions = {}): Promise<void> {
+    await Promise.all(this.items.map((replica) => replica.run(options)));
+  }
+
+  /** Close releases all replicas' internal connections. */
+  close(): void {
+    for (const replica of this.items) replica.close();
+    this.items = [];
+  }
+}

package/src/util/fs.ts

@@ -0,0 +1,82 @@
+/** File-system helpers for atomic writes and directory fsync. */
+
+import { writeSync } from 'node:fs';
+import { mkdir, open, type FileHandle } from 'node:fs/promises';
+import { dirname } from 'node:path';
+import type { ByteSink } from '../ltx/sink.ts';
+
+/** Ensures a directory (and parents) exists; no-op if already present. */
+export async function mkdirAll(path: string): Promise<void> {
+  await mkdir(path, { recursive: true });
+}
+
+/** Fsyncs a directory so a preceding rename within it is durable. */
+export async function fsyncDir(path: string): Promise<void> {
+  const handle = await open(path, 'r');
+  try {
+    await handle.sync();
+  } finally {
+    await handle.close();
+  }
+}
+
+/** Returns the parent directory of a path (portable). */
+export function parentDir(path: string): string {
+  return dirname(path);
+}
+
+/**
+ * A file-backed {@link ByteSink}. It batches bytes into fixed-size blocks and
+ * writes them synchronously, so the LTX encoder can stream a large file to disk
+ * without buffering the whole file (or the whole database) in memory.
+ */
+export class FileSink implements ByteSink {
+  private fh: FileHandle;
+  private buf: Uint8Array;
+  private pos = 0;
+  private filePos = 0;
+
+  constructor(fh: FileHandle, chunkSize = 256 * 1024) {
+    this.fh = fh;
+    this.buf = new Uint8Array(chunkSize);
+  }
+
+  write(bytes: Uint8Array): void {
+    let o = 0;
+    while (o < bytes.length) {
+      if (this.pos === this.buf.length) this.flushSync();
+      const take = Math.min(this.buf.length - this.pos, bytes.length - o);
+      this.buf.set(bytes.subarray(o, o + take), this.pos);
+      this.pos += take;
+      o += take;
+    }
+  }
+
+  /** Returns the number of bytes written so far. */
+  get length(): number {
+    return this.filePos + this.pos;
+  }
+
+  private flushSync(): void {
+    if (this.pos === 0) return;
+    writeAllSync(this.fh, this.buf, this.pos, this.filePos);
+    this.filePos += this.pos;
+    this.pos = 0;
+  }
+
+  /** Flushes any remaining buffered bytes and fsyncs the file. */
+  async flush(): Promise<void> {
+    this.flushSync();
+    await this.fh.sync();
+  }
+}
+
+/** Writes all of `buf[0..len)` to `handle`'s fd starting at `position`. */
+function writeAllSync(handle: FileHandle, buf: Uint8Array, len: number, position: number): void {
+  const fd = handle.fd;
+  let offset = 0;
+  while (offset < len) {
+    const written = writeSync(fd, buf, offset, len - offset, position + offset);
+    offset += written;
+  }
+}

package/src/util/logger.ts

@@ -0,0 +1,15 @@
+/** Minimal structured logger interface (stand-in for Go's slog.Logger). */
+export interface Logger {
+  debug(msg: string, ...args: unknown[]): void;
+  info(msg: string, ...args: unknown[]): void;
+  warn(msg: string, ...args: unknown[]): void;
+  error(msg: string, ...args: unknown[]): void;
+}
+
+/** A no-op logger used when none is provided. */
+export const noopLogger: Logger = {
+  debug() {},
+  info() {},
+  warn() {},
+  error() {},
+};

package/src/util/stream.ts

@@ -0,0 +1,28 @@
+/** Stream helpers for consuming/capturing LTX byte streams. */
+
+/** Concatenates an array of byte chunks into a single Uint8Array. */
+export function concatBytes(chunks: readonly Uint8Array[]): Uint8Array {
+  let total = 0;
+  for (const c of chunks) total += c.length;
+  const out = new Uint8Array(total);
+  let o = 0;
+  for (const c of chunks) {
+    out.set(c, o);
+    o += c.length;
+  }
+  return out;
+}
+
+/**
+ * Reads an entire async iterable of byte chunks into a single Uint8Array.
+ * String chunks are encoded as UTF-8.
+ */
+export async function readAll(
+  stream: AsyncIterable<Uint8Array | string> | { [Symbol.asyncIterator](): AsyncIterator<Uint8Array | string> },
+): Promise<Uint8Array> {
+  const chunks: Uint8Array[] = [];
+  for await (const chunk of stream) {
+    chunks.push(typeof chunk === 'string' ? new TextEncoder().encode(chunk) : Uint8Array.from(chunk));
+  }
+  return concatBytes(chunks);
+}

package/src/wal/checksum.ts

@@ -0,0 +1,39 @@
+/**
+ * SQLite WAL checksum.
+ *
+ * Matches litestream's `WALChecksum` and SQLite's wal-checksum algorithm.
+ * The byte order is chosen from the WAL magic: 0x377f0682 => little-endian,
+ * 0x377f0683 => big-endian. The checksum runs over 8-byte units:
+ *   s0 += u32[a] + s1 ; s1 += u32[a+4] + s0
+ */
+
+import { readU32BE, readU32LE } from '../ltx/bytes.ts';
+import { WALFormatError } from '../errors.ts';
+
+export type ByteOrder = 'le' | 'be';
+
+/** Reads a u32 from `buf` at `off` using the given byte order. */
+export function readWalU32(bo: ByteOrder, buf: Uint8Array, off: number): number {
+  if (bo === 'le') return readU32LE(buf, off);
+  return readU32BE(buf, off);
+}
+
+/**
+ * walChecksum computes a running SQLite WAL checksum over a byte slice whose
+ * length is a multiple of 8. Returns the updated (s0, s1) pair.
+ */
+export function walChecksum(
+  bo: ByteOrder,
+  s0: number,
+  s1: number,
+  b: Uint8Array,
+): [number, number] {
+  if (b.length % 8 !== 0) {
+    throw new WALFormatError('misaligned checksum byte slice');
+  }
+  for (let i = 0; i < b.length; i += 8) {
+    s0 = (s0 + readWalU32(bo, b, i) + s1) >>> 0;
+    s1 = (s1 + readWalU32(bo, b, i + 4) + s0) >>> 0;
+  }
+  return [s0, s1];
+}

package/src/wal/constants.ts

@@ -0,0 +1,10 @@
+/** SQLite WAL (Write-Ahead Log) file constants. */
+
+export const WALHeaderSize = 32;
+export const WALFrameHeaderSize = 24;
+
+/** WAL magic values (always read big-endian; determines checksum byte order). */
+export const WALMagicLE = 0x377f0682; // checksums are computed little-endian
+export const WALMagicBE = 0x377f0683; // checksums are computed big-endian
+
+export const WALVersion = 3007000;

package/src/wal/index.ts

@@ -0,0 +1,5 @@
+/** SQLite WAL module — public surface for reading WAL files. */
+
+export * from './constants.ts';
+export * from './checksum.ts';
+export * from './reader.ts';

package/src/wal/reader.ts

@@ -0,0 +1,264 @@
+/**
+ * WALReader parses a SQLite WAL file and verifies salt & checksum integrity
+ * while reading. It does not enforce transaction boundaries; the caller is
+ * responsible for handling uncommitted frames.
+ *
+ * Mirrors litestream's `WALReader` (wal_reader.go).
+ */
+
+import { WALFrameHeaderSize, WALHeaderSize, WALMagicBE, WALMagicLE, WALVersion } from './constants.ts';
+import type { ByteOrder } from './checksum.ts';
+import { walChecksum } from './checksum.ts';
+import { readU32BE } from '../ltx/bytes.ts';
+import { WALFormatError } from '../errors.ts';
+
+/** Raised when the WAL ends, or when a salt/checksum mismatch is detected. */
+export class WALEOFError extends Error {}
+
+/** Raised when starting mid-WAL and the previous frame does not match. */
+export class PrevFrameMismatchError extends Error {
+  readonly cause: Error;
+  constructor(cause: Error) {
+    super(`prev frame mismatch: ${cause.message}`);
+    this.cause = cause;
+  }
+}
+
+export interface WALFrame {
+  pgno: number;
+  commit: number;
+}
+
+export interface WALPageMap {
+  /** Latest committed offset (file offset) for each page. */
+  pageIndex: Map<number, number>;
+  /** Offset just past the last read frame. */
+  maxOffset: number;
+  /** Final database size in pages (from the last commit frame). */
+  commit: number;
+  /** Whether the scan was bounded by maxBytes. */
+  limited: boolean;
+}
+
+/** WALReader reads and validates frames from an in-memory WAL byte buffer. */
+export class WALReader {
+  private data: Uint8Array;
+  private bo!: ByteOrder;
+  private pageSizeVal = 0;
+  private seqVal = 0;
+  private salt1Value = 0;
+  private salt2Value = 0;
+  private chksum1 = 0;
+  private chksum2 = 0;
+  private frameN = 0;
+
+  constructor(data: Uint8Array) {
+    this.data = data;
+    this.readHeader();
+  }
+
+  /** Creates a reader positioned at `offset`, loading salts from the caller. */
+  static withOffset(data: Uint8Array, offset: number, salt1: number, salt2: number): WALReader {
+    if (offset <= WALHeaderSize) {
+      throw new WALFormatError(
+        `offset (${offset}) must be greater than the wal header size (${WALHeaderSize})`,
+      );
+    }
+    const r = new WALReader(data);
+
+    // Load the salt from the caller; the beginning of the file may be gone.
+    r.salt1Value = salt1;
+    r.salt2Value = salt2;
+
+    // Ensure the offset is positioned on a frame start.
+    const frameSize = r.pageSizeVal + WALFrameHeaderSize;
+    if ((offset - WALHeaderSize) % frameSize !== 0) {
+      throw new WALFormatError(`unaligned wal offset ${offset} for page size ${r.pageSizeVal}`);
+    }
+    r.frameN = Math.floor((offset - WALHeaderSize) / frameSize);
+
+    // Read the previous page to load the checksum state.
+    r.frameN--;
+    try {
+      r.readFrameInternal(new Uint8Array(r.pageSizeVal), false);
+    } catch (err) {
+      if (err instanceof WALEOFError) {
+        throw new PrevFrameMismatchError(err);
+      }
+      throw err;
+    }
+
+    return r;
+  }
+
+  /** PageSize returns the page size from the header. */
+  pageSize(): number {
+    return this.pageSizeVal;
+  }
+
+  /** Seq returns the checkpoint sequence from the header. */
+  seq(): number {
+    return this.seqVal;
+  }
+
+  /** Salt1 returns the header salt-1 value. */
+  salt1(): number {
+    return this.salt1Value;
+  }
+
+  /** Salt2 returns the header salt-2 value. */
+  salt2(): number {
+    return this.salt2Value;
+  }
+
+  /** Offset returns the file offset of the last read frame (0 if none). */
+  offset(): number {
+    if (this.frameN === 0) return 0;
+    return WALHeaderSize + (this.frameN - 1) * (WALFrameHeaderSize + this.pageSizeVal);
+  }
+
+  /** ReadFrame reads the next frame, verifying salt & checksum. */
+  readFrame(data: Uint8Array): WALFrame {
+    return this.readFrameInternal(data, true);
+  }
+
+  /** PageMap scans committed frames and returns the latest page offsets. */
+  pageMap(maxBytes = 0): WALPageMap {
+    const m = new Map<number, number>();
+    const txMap = new Map<number, number>();
+    const data = new Uint8Array(this.pageSizeVal);
+    const frameSize = WALFrameHeaderSize + this.pageSizeVal;
+    const startOffset = WALHeaderSize + this.frameN * frameSize;
+    let commit = 0;
+    let limited = false;
+
+    for (;;) {
+      let frame: WALFrame;
+      try {
+        frame = this.readFrame(data);
+      } catch (err) {
+        if (err instanceof WALEOFError) break;
+        throw err;
+      }
+
+      // Buffer the latest offset for each page in the current transaction.
+      const offset = this.offset();
+      txMap.set(frame.pgno, offset);
+
+      // On a commit record, flush pending pages into the full map.
+      if (frame.commit !== 0) {
+        for (const [pgno, off] of txMap) m.set(pgno, off);
+        txMap.clear();
+        commit = frame.commit;
+
+        if (maxBytes > 0 && this.offset() + frameSize - startOffset >= maxBytes) {
+          limited = true;
+          break;
+        }
+      }
+    }
+
+    // Drop pages beyond the final commit size (DB may have shrunk via VACUUM).
+    for (const pgno of [...m.keys()]) {
+      if (pgno > commit) m.delete(pgno);
+    }
+
+    if (m.size === 0) {
+      return { pageIndex: m, maxOffset: 0, commit: 0, limited };
+    }
+
+    // Compute the highest frame offset and extend past its data.
+    let end = 0;
+    for (const offset of m.values()) {
+      if (end === 0 || offset > end) end = offset;
+    }
+    end += frameSize;
+
+    return { pageIndex: m, maxOffset: end, commit, limited };
+  }
+
+  private readHeader(): void {
+    if (this.data.length < WALHeaderSize) {
+      throw new WALEOFError();
+    }
+    const hdr = this.data.subarray(0, WALHeaderSize);
+
+    const magic = readU32BE(hdr, 0);
+    if (magic === WALMagicLE) this.bo = 'le';
+    else if (magic === WALMagicBE) this.bo = 'be';
+    else throw new WALFormatError(`invalid wal header magic: ${magic.toString(16)}`);
+
+    const chksum1 = readU32BE(hdr, 24);
+    const chksum2 = readU32BE(hdr, 28);
+    const [v0, v1] = walChecksum(this.bo, 0, 0, hdr.subarray(0, 24));
+    if (v0 !== chksum1 || v1 !== chksum2) {
+      throw new WALEOFError();
+    }
+
+    const version = readU32BE(hdr, 4);
+    if (version !== WALVersion) {
+      throw new WALFormatError(`unsupported wal version: ${version}`);
+    }
+
+    this.pageSizeVal = readU32BE(hdr, 8);
+    this.seqVal = readU32BE(hdr, 12);
+    this.salt1Value = readU32BE(hdr, 16);
+    this.salt2Value = readU32BE(hdr, 20);
+    this.chksum1 = chksum1;
+    this.chksum2 = chksum2;
+  }
+
+  private readFrameInternal(data: Uint8Array, verifyChecksum: boolean): WALFrame {
+    if (data.length !== this.pageSizeVal) {
+      throw new WALFormatError(
+        `buffer size (${data.length}) must match page size (${this.pageSizeVal})`,
+      );
+    }
+
+    const frameSize = this.pageSizeVal + WALFrameHeaderSize;
+    const offset = WALHeaderSize + this.frameN * frameSize;
+    if (offset + frameSize > this.data.length) {
+      throw new WALEOFError();
+    }
+
+    const hdr = this.data.subarray(offset, offset + WALFrameHeaderSize);
+    const page = this.data.subarray(offset + WALFrameHeaderSize, offset + frameSize);
+
+    // Verify the frame salt matches the header salt.
+    const salt1 = readU32BE(hdr, 8);
+    const salt2 = readU32BE(hdr, 12);
+    if (this.salt1Value !== salt1 || this.salt2Value !== salt2) {
+      throw new WALEOFError();
+    }
+
+    const fchksum1 = readU32BE(hdr, 16);
+    const fchksum2 = readU32BE(hdr, 20);
+
+    if (verifyChecksum) {
+      [this.chksum1, this.chksum2] = walChecksum(
+        this.bo,
+        this.chksum1,
+        this.chksum2,
+        hdr.subarray(0, 8),
+      );
+      [this.chksum1, this.chksum2] = walChecksum(
+        this.bo,
+        this.chksum1,
+        this.chksum2,
+        page,
+      );
+      if (this.chksum1 !== fchksum1 || this.chksum2 !== fchksum2) {
+        throw new WALEOFError();
+      }
+    } else {
+      this.chksum1 = fchksum1;
+      this.chksum2 = fchksum2;
+    }
+
+    const pgno = readU32BE(hdr, 0);
+    const commit = readU32BE(hdr, 4);
+    this.frameN++;
+
+    return { pgno, commit };
+  }
+}