@minnowdb/core 0.2.1 → 0.4.0

package/dist/storage/snapshot.d.ts

@@ -1,108 +1,32 @@
-import type { FtsPosting, SegmentRecord, TableRecord, TransactionRecord } from "./types.js";
+/** Public snapshot API over the bounded framed v1 container. */
+export * from "./snapshot-stream.js";
 export declare const SNAPSHOT_FORMAT_VERSION = 1;
-/** One indexed column's full-text base, as it stood at the snapshot's version. */
-export interface SnapshotFtsIndex {
-    columnId: string;
-    coversVersion: number;
-    totalTokens: number;
-    /** Term-range partitioned, term-sorted within each chunk. */
-    chunks: FtsPosting[][];
-}
-export interface SnapshotTable {
-    record: TableRecord;
-    /** The next row ID a write should reserve; below it, IDs may already be in use. */
-    nextRowId: bigint;
-    autoIncrement: Array<{
-        columnId: string;
-        next: bigint;
-    }>;
-    /** Complete unique-key membership, so a write after loading still conflicts correctly. */
-    uniqueKeyTokens: string[];
-    fts: SnapshotFtsIndex[];
-}
-export interface DatabaseSnapshot {
-    /** The manifest version this snapshot captured, which the restored database keeps. */
-    version: number;
-    createdAt: string;
-    tables: SnapshotTable[];
-    segments: SegmentRecord[];
-    /** Committed transactions only; segment visibility resolves through `committedVersion`. */
-    transactions: TransactionRecord[];
-    blocks: Array<{
-        id: string;
-        bytes: Uint8Array;
-    }>;
-}
-/** Where a snapshot load has got to, for a progress bar over a multi-megabyte file. */
+/** Export and worker-transport chunk ceiling; direct imports accept any non-empty source chunk. */
+export declare const MAX_SNAPSHOT_STREAM_CHUNK_BYTES: number;
 export interface SnapshotLoadProgress {
     phase: "blocks" | "catalog" | "done";
     writtenBytes: number;
     totalBytes: number;
 }
-/**
- * Where a snapshot export has got to. `reading` covers copying the version out of the store and
- * encoding it, and reports no byte counts because the encoded length is not known until it is
- * finished. `transfer` happens only when the database lives in a worker and the bytes are handed
- * back a chunk at a time; an in-page database goes straight from `reading` to `done`.
- */
 export interface SnapshotExportProgress {
     phase: "reading" | "transfer" | "done";
     transferredBytes: number;
-    /** Zero while reading; the encoded length once it is known. */
+    /** Zero while streaming; the exact encoded length is known by the byte-array wrapper. */
     totalBytes: number;
 }
-/** What a header-only read reports, without touching a single block payload. */
+/** Header summary used by restore confirmation UIs without retaining database metadata. */
 export interface SnapshotSummary {
     formatVersion: number;
     version: number;
     createdAt: string;
     tableCount: number;
     blockCount: number;
-    /** Total byte length of the block payloads, excluding the header. */
+    /** Stored body payload bytes, excluding frame envelopes, header, and footer. */
     payloadBytes: number;
     byteLength: number;
 }
 /**
- * Serializes a snapshot. Block bytes are copied verbatim — they are already compressed and
- * checksummed, so the container never re-encodes or re-compresses them. The header is gzipped,
- * because it is JSON and its bulk is full-text posting lists and segment records, which compress
- * by roughly four to one.
- */
-export declare function encodeSnapshot(snapshot: DatabaseSnapshot): Promise<Uint8Array>;
-/**
- * Reads only the header. Cheap enough to run against a multi-hundred-megabyte file before
- * deciding whether to load it.
+ * Reads and validates the canonical header from a materialized snapshot. The decoder closes its
+ * source immediately after the header; body checks run during import, where frames are consumed.
  */
 export declare function readSnapshotSummary(bytes: Uint8Array): Promise<SnapshotSummary>;
-/**
- * Parses a snapshot and authenticates every block. `inspectBlock` verifies each payload's magic,
- * version, declared lengths, and envelope checksum without decompressing it, so a corrupt file
- * fails here rather than mid-query.
- */
-export declare function decodeSnapshot(bytes: Uint8Array): Promise<DatabaseSnapshot>;
-/**
- * Shared by both stores' exporters: given the manifest's live block set, decide which segments
- * and transactions the snapshot needs.
- *
- * A segment is live when every block it references survives in the manifest — compaction
- * supersedes a segment by dropping its blocks, so that test is exactly the reader's. A
- * transaction is needed when a live segment names it, and only committed transactions qualify:
- * an active one's artifacts are unpublished and would dangle. Their pending lists are cleared,
- * because after the restore the manifest is the only root anything needs.
- *
- * `committedVersion` is preserved exactly, and the restored database keeps the captured version
- * number rather than renumbering to 1. Segments sort by `logicalOrder ?? committedVersion`, so
- * flattening those versions would let an update fold ahead of the insert it patches; and change
- * detection compares committed versions against a snapshot version, which would misread any
- * transaction numbered above the manifest.
- */
-export declare function selectLiveRecords(input: {
-    liveBlockIds: ReadonlySet<string>;
-    segments: readonly SegmentRecord[];
-    transactions: readonly TransactionRecord[];
-    version: number;
-}): {
-    segments: SegmentRecord[];
-    transactions: TransactionRecord[];
-};
-//# sourceMappingURL=snapshot.d.ts.map

package/dist/storage/snapshot.js

@@ -1,276 +1,44 @@
-/**
- * Database snapshots: one committed version of a database, copied out as a single portable
- * byte array and loaded back into any block store.
- *
- * A snapshot is a *published* view, not a backup of a store's internals. It carries the block
- * bytes the current manifest points at, the catalog needed to read them (tables, segments, and
- * the committed transactions segment visibility resolves through), the counters a later write
- * has to continue from, and the unique-key membership a later write has to conflict against.
- * It deliberately drops everything transient — leases, spill pages, garbage-collection and
- * compaction jobs, in-flight transactions — and it drops version history: the snapshot restores
- * as a single checkpoint, so a 10,000-commit database loads as one clean version.
- *
- * The container is a small JSON header followed by the block payloads laid end to end. Blocks
- * are already self-describing and doubly CRC-checked (see `../block-format/block.ts`), so the
- * container adds only a checksum over its own header and lets `inspectBlock` authenticate each
- * payload without decompressing any of them.
- */
-import { crc32, gzipCodec, inspectBlock } from "../block-format/index.js";
-/** ASCII `MNWSNAP1`, matching the block format's habit of a readable magic. */
-const MAGIC = Uint8Array.from([0x4d, 0x4e, 0x57, 0x53, 0x4e, 0x41, 0x50, 0x31]);
-/**
- * magic(8) + formatVersion(4) + storedHeaderLength(4) + headerChecksum(4) + headerLength(4)
- * + headerCodec(4). All little-endian, matching the block format.
- */
-const PREFIX_LENGTH = 28;
-const HEADER_CODEC_NONE = 0;
-const HEADER_CODEC_GZIP = 1;
+/** Public snapshot API over the bounded framed v1 container. */
+export * from "./snapshot-stream.js";
+import { SNAPSHOT_FRAME_KINDS } from "./types.js";
+import { decodeSnapshotFrameStream } from "./snapshot-stream.js";
 export const SNAPSHOT_FORMAT_VERSION = 1;
-function encodeSegment(segment) {
-    const { rowIdStart, rowIdEndExclusive, rowIdSpans, ...rest } = segment;
-    return {
-        ...rest,
-        rowIdStart: rowIdStart.toString(),
-        rowIdEndExclusive: rowIdEndExclusive.toString(),
-        ...(rowIdSpans === undefined
-            ? {}
-            : {
-                rowIdSpans: rowIdSpans.map((span) => ({
-                    rowStart: span.rowStart,
-                    rowCount: span.rowCount,
-                    rowIdStart: span.rowIdStart.toString(),
-                })),
-            }),
-    };
-}
-function decodeSegment(segment) {
-    const { rowIdStart, rowIdEndExclusive, rowIdSpans, ...rest } = segment;
-    const spans = rowIdSpans?.map((span) => ({
-        rowStart: span.rowStart,
-        rowCount: span.rowCount,
-        rowIdStart: parseBigInt(span.rowIdStart, "segment row-ID span"),
-    }));
-    return {
-        ...rest,
-        rowIdStart: parseBigInt(rowIdStart, "segment row-ID start"),
-        rowIdEndExclusive: parseBigInt(rowIdEndExclusive, "segment row-ID end"),
-        ...(spans === undefined ? {} : { rowIdSpans: spans }),
-    };
-}
-function encodePosting(posting) {
-    const [first, ...rest] = posting.rowIds;
-    if (first === undefined)
-        return { term: posting.term, first: "0", deltas: [], tf: posting.tf };
-    const deltas = [];
-    let previous = first;
-    for (const rowId of rest) {
-        const gap = rowId - previous;
-        if (gap > BigInt(Number.MAX_SAFE_INTEGER)) {
-            throw new RangeError("Full-text row-ID gap is too large to snapshot");
-        }
-        deltas.push(Number(gap));
-        previous = rowId;
-    }
-    return { term: posting.term, first: first.toString(), deltas, tf: posting.tf };
-}
-function decodePosting(posting) {
-    const rowIds = [];
-    if (posting.tf.length > 0) {
-        let current = parseBigInt(posting.first, "full-text row ID");
-        rowIds.push(current);
-        for (const delta of posting.deltas) {
-            current += BigInt(delta);
-            rowIds.push(current);
-        }
-    }
-    return { term: posting.term, rowIds, tf: posting.tf };
-}
-function parseBigInt(value, context) {
-    if (!/^-?\d+$/.test(value))
-        throw new Error(`Snapshot has an invalid ${context}: ${value}`);
-    return BigInt(value);
-}
+/** Export and worker-transport chunk ceiling; direct imports accept any non-empty source chunk. */
+export const MAX_SNAPSHOT_STREAM_CHUNK_BYTES = 1024 * 1024;
 /**
- * Serializes a snapshot. Block bytes are copied verbatim — they are already compressed and
- * checksummed, so the container never re-encodes or re-compresses them. The header is gzipped,
- * because it is JSON and its bulk is full-text posting lists and segment records, which compress
- * by roughly four to one.
- */
-export async function encodeSnapshot(snapshot) {
-    const header = {
-        formatVersion: SNAPSHOT_FORMAT_VERSION,
-        version: snapshot.version,
-        createdAt: snapshot.createdAt,
-        blocks: snapshot.blocks.map((block) => ({ id: block.id, length: block.bytes.byteLength })),
-        tables: snapshot.tables.map((table) => ({
-            record: table.record,
-            nextRowId: table.nextRowId.toString(),
-            autoIncrement: table.autoIncrement.map((entry) => ({
-                columnId: entry.columnId,
-                next: entry.next.toString(),
-            })),
-            uniqueKeyTokens: table.uniqueKeyTokens,
-            fts: table.fts.map((index) => ({
-                columnId: index.columnId,
-                coversVersion: index.coversVersion,
-                totalTokens: index.totalTokens,
-                chunks: index.chunks.map((chunk) => chunk.map(encodePosting)),
-            })),
-        })),
-        segments: snapshot.segments.map(encodeSegment),
-        transactions: snapshot.transactions,
-    };
-    const json = new TextEncoder().encode(JSON.stringify(header));
-    const stored = await gzipCodec.compress(json);
-    const payloadBytes = snapshot.blocks.reduce((total, block) => total + block.bytes.byteLength, 0);
-    const bytes = new Uint8Array(PREFIX_LENGTH + stored.byteLength + payloadBytes);
-    bytes.set(MAGIC, 0);
-    const view = new DataView(bytes.buffer);
-    view.setUint32(8, SNAPSHOT_FORMAT_VERSION, true);
-    view.setUint32(12, stored.byteLength, true);
-    view.setUint32(16, crc32(stored), true);
-    view.setUint32(20, json.byteLength, true);
-    view.setUint32(24, HEADER_CODEC_GZIP, true);
-    bytes.set(stored, PREFIX_LENGTH);
-    let offset = PREFIX_LENGTH + stored.byteLength;
-    for (const block of snapshot.blocks) {
-        bytes.set(block.bytes, offset);
-        offset += block.bytes.byteLength;
-    }
-    return bytes;
-}
-async function readHeader(bytes) {
-    if (bytes.byteLength < PREFIX_LENGTH)
-        throw new Error("Truncated snapshot header");
-    if (!MAGIC.every((byte, index) => bytes[index] === byte)) {
-        throw new Error("Not a Minnow snapshot");
-    }
-    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
-    const formatVersion = view.getUint32(8, true);
-    if (formatVersion !== SNAPSHOT_FORMAT_VERSION) {
-        throw new Error(`Unsupported snapshot version ${String(formatVersion)}`);
-    }
-    const storedLength = view.getUint32(12, true);
-    const payloadOffset = PREFIX_LENGTH + storedLength;
-    if (payloadOffset > bytes.byteLength)
-        throw new Error("Truncated snapshot header");
-    const stored = bytes.subarray(PREFIX_LENGTH, payloadOffset);
-    // Checked before the header is decompressed, let alone parsed: a damaged header must fail
-    // as a checksum mismatch rather than as whatever gzip or JSON happens to say about it.
-    if (crc32(stored) !== view.getUint32(16, true)) {
-        throw new Error("Snapshot header checksum mismatch");
-    }
-    const codec = view.getUint32(24, true);
-    if (codec !== HEADER_CODEC_NONE && codec !== HEADER_CODEC_GZIP) {
-        throw new Error(`Unsupported snapshot header codec ${String(codec)}`);
-    }
-    const json = codec === HEADER_CODEC_GZIP
-        ? await gzipCodec.decompress(stored, view.getUint32(20, true))
-        : stored;
-    const header = JSON.parse(new TextDecoder().decode(json));
-    return { header, payloadOffset };
-}
-/**
- * Reads only the header. Cheap enough to run against a multi-hundred-megabyte file before
- * deciding whether to load it.
+ * Reads and validates the canonical header from a materialized snapshot. The decoder closes its
+ * source immediately after the header; body checks run during import, where frames are consumed.
  */
 export async function readSnapshotSummary(bytes) {
-    const { header, payloadOffset } = await readHeader(bytes);
-    return {
-        formatVersion: header.formatVersion,
-        version: header.version,
-        createdAt: header.createdAt,
-        tableCount: header.tables.length,
-        blockCount: header.blocks.length,
-        payloadBytes: bytes.byteLength - payloadOffset,
-        byteLength: bytes.byteLength,
-    };
-}
-/**
- * Parses a snapshot and authenticates every block. `inspectBlock` verifies each payload's magic,
- * version, declared lengths, and envelope checksum without decompressing it, so a corrupt file
- * fails here rather than mid-query.
- */
-export async function decodeSnapshot(bytes) {
-    const { header, payloadOffset } = await readHeader(bytes);
-    const declared = header.blocks.reduce((total, block) => total + block.length, 0);
-    if (payloadOffset + declared !== bytes.byteLength) {
-        throw new Error("Snapshot length mismatch");
-    }
-    const blocks = [];
-    let offset = payloadOffset;
-    for (const block of header.blocks) {
-        const payload = bytes.subarray(offset, offset + block.length);
-        inspectBlock(payload);
-        blocks.push({ id: block.id, bytes: payload });
-        offset += block.length;
+    if (!(bytes instanceof Uint8Array) || bytes.byteLength === 0) {
+        throw new TypeError("Snapshot bytes must be a non-empty Uint8Array");
     }
-    const present = new Set(blocks.map((block) => block.id));
-    const segments = header.segments.map(decodeSegment);
-    for (const segment of segments) {
-        for (const ids of Object.values(segment.columnBlockIds)) {
-            for (const id of ids) {
-                if (!present.has(id))
-                    throw new Error(`Snapshot segment references a missing block: ${id}`);
-            }
+    const entries = decodeSnapshotFrameStream((async function* () {
+        for (let offset = 0; offset < bytes.byteLength; offset += MAX_SNAPSHOT_STREAM_CHUNK_BYTES) {
+            yield bytes.subarray(offset, offset + MAX_SNAPSHOT_STREAM_CHUNK_BYTES);
         }
+    })());
+    const first = await entries.next();
+    await entries.return(undefined);
+    if (first.done || first.value.type !== "header")
+        throw new Error("Snapshot header is missing");
+    return summaryFromHeader(first.value.header, bytes.byteLength);
+}
+function summaryFromHeader(header, byteLength) {
+    let payloadBytes = 0;
+    for (const kind of SNAPSHOT_FRAME_KINDS) {
+        const next = payloadBytes + header.kinds[kind].storedBytes;
+        if (!Number.isSafeInteger(next))
+            throw new RangeError("Snapshot payload is too large");
+        payloadBytes = next;
     }
     return {
-        version: header.version,
+        formatVersion: header.formatVersion,
+        version: header.databaseVersion,
         createdAt: header.createdAt,
-        tables: header.tables.map((table) => ({
-            record: table.record,
-            nextRowId: parseBigInt(table.nextRowId, "row-ID counter"),
-            autoIncrement: table.autoIncrement.map((entry) => ({
-                columnId: entry.columnId,
-                next: parseBigInt(entry.next, "auto-increment counter"),
-            })),
-            uniqueKeyTokens: table.uniqueKeyTokens,
-            fts: table.fts.map((index) => ({
-                columnId: index.columnId,
-                coversVersion: index.coversVersion,
-                totalTokens: index.totalTokens,
-                chunks: index.chunks.map((chunk) => chunk.map(decodePosting)),
-            })),
-        })),
-        segments,
-        transactions: header.transactions,
-        blocks,
+        tableCount: header.kinds["catalog-page"].itemCount,
+        blockCount: header.kinds.block.itemCount,
+        payloadBytes,
+        byteLength,
     };
 }
-/**
- * Shared by both stores' exporters: given the manifest's live block set, decide which segments
- * and transactions the snapshot needs.
- *
- * A segment is live when every block it references survives in the manifest — compaction
- * supersedes a segment by dropping its blocks, so that test is exactly the reader's. A
- * transaction is needed when a live segment names it, and only committed transactions qualify:
- * an active one's artifacts are unpublished and would dangle. Their pending lists are cleared,
- * because after the restore the manifest is the only root anything needs.
- *
- * `committedVersion` is preserved exactly, and the restored database keeps the captured version
- * number rather than renumbering to 1. Segments sort by `logicalOrder ?? committedVersion`, so
- * flattening those versions would let an update fold ahead of the insert it patches; and change
- * detection compares committed versions against a snapshot version, which would misread any
- * transaction numbered above the manifest.
- */
-export function selectLiveRecords(input) {
-    const committed = new Map(input.transactions
-        .filter((record) => record.status === "committed" &&
-        record.committedVersion !== null &&
-        record.committedVersion <= input.version)
-        .map((record) => [record.id, record]));
-    const segments = input.segments.filter((segment) => {
-        if (!committed.has(segment.transactionId))
-            return false;
-        return Object.values(segment.columnBlockIds)
-            .flat()
-            .every((id) => input.liveBlockIds.has(id));
-    });
-    const usedTransactionIds = new Set(segments.map((segment) => segment.transactionId));
-    const transactions = [...committed.values()]
-        .filter((record) => usedTransactionIds.has(record.id))
-        .map((record) => ({ ...record, pendingBlockIds: [], pendingSegmentIds: [] }));
-    return { segments, transactions };
-}
-//# sourceMappingURL=snapshot.js.map

package/dist/storage/toolkit/extents.d.ts

@@ -1,13 +1,13 @@
-import type { SyncFileHandle } from "./sync-file.js";
+import { type SyncFileHandle } from "./sync-file.js";
 /**
  * Packed, append-only extent files for bulk bytes — blocks and full-text base chunks.
  *
  * The leader appends into one tail extent through a handle it keeps open (microseconds per
  * write) and seals it at a size threshold; sealed extents are immutable. Every payload's home
- * is a placement — extent id, offset, length — recorded by the WAL entry that publishes it,
- * so recovery re-derives the tail position from the replayed placements and any bytes beyond
- * it (a crash between an extent write and its WAL frame) are dead space, overwritten by the
- * next append.
+ * is a placement — extent id, offset, length, and whole-payload checksum — recorded by the WAL
+ * entry that publishes it. Recovery verifies each new placement before applying its entry,
+ * re-derives the tail position from the consistent prefix, and overwrites any unpublished bytes
+ * beyond it on the next append.
  *
  * Extents assume a single writer (the OPFS adapter's leader), so holding handles — the tail's
  * permanently, sealed ones' in a small read cache — never contends with anyone.
@@ -22,10 +22,16 @@ export interface ExtentFiles {
     }): Promise<SyncFileHandle>;
     deleteFile(path: readonly string[]): Promise<boolean>;
 }
+export interface ExtentPoolOptions {
+    /** @internal Deterministic checksum instrumentation for performance/correctness tests. */
+    _checksumForTests?: (bytes: Uint8Array) => number;
+}
 export interface Placement {
     extent: number;
     offset: number;
     length: number;
+    /** CRC32 of every stored payload byte, independent of the payload's own format. */
+    checksum: number;
 }
 /** Persisted in checkpoints; placements in replayed WAL entries refine `tailOffset`. */
 export interface ExtentMeta {
@@ -35,11 +41,31 @@ export interface ExtentMeta {
     /** Live payload bytes per extent id; an extent at zero that is not the tail is deletable. */
     liveBytes: Array<readonly [number, number]>;
 }
+/** Opaque O(1) rollback point for bytes not yet published by a WAL frame. */
+export declare class ExtentBatchMark {
+    readonly tailExtentId: number;
+    readonly nextExtentId: number;
+    readonly tailOffset: number;
+    /** @internal Original counters are captured lazily only for extents the batch touches. */
+    readonly _liveBytesBefore: Map<number, number | undefined>;
+    constructor(tailExtentId: number, nextExtentId: number, tailOffset: number);
+}
 export declare class ExtentPool {
     #private;
     private constructor();
     /** Opens (or creates) the tail extent. `meta` comes from the checkpoint, if any. */
-    static open(tree: ExtentFiles, meta: ExtentMeta | undefined): Promise<ExtentPool>;
+    static open(tree: ExtentFiles, meta: ExtentMeta | undefined, options?: ExtentPoolOptions): Promise<ExtentPool>;
+    /** Captures the exact append/accounting state before an unpublished multi-payload batch. */
+    markBatch(): ExtentBatchMark;
+    /** Publishes a successful batch; constant-time and allocation-free. */
+    commitBatch(mark: ExtentBatchMark): void;
+    /**
+     * Rolls back every append and accounting mutation since `mark`. This is deliberately an
+     * async slow path; successful appends retain the allocation-free hot path.
+     */
+    rollbackBatch(mark: ExtentBatchMark): Promise<void>;
+    /** Removes an unpublished physical suffix left by a crash before its WAL publication. */
+    truncateTailToPublishedOffset(): void;
     /**
      * Replay hook: a WAL entry recorded this placement after the checkpoint was taken. Advances
      * the tail position and live-byte accounting to match what the writer had done.
@@ -51,12 +77,18 @@ export declare class ExtentPool {
     /** Appends synchronously; seals and rolls the tail first when it is full (the async part). */
     append(bytes: Uint8Array, flush: boolean): Promise<Placement>;
     read(placement: Placement): Promise<Uint8Array>;
+    /** Reads and verifies one complete opaque payload with exactly one checksum scan. */
+    readVerified(placement: Placement): Promise<Uint8Array>;
+    /** True when the complete placement currently exists; used to bound crash replay. */
+    contains(placement: Placement): Promise<boolean>;
     /**
      * Returns the sealed extent with the most dead space once less than half its physical bytes
      * remain live. Relocating that extent keeps sealed-file bytes below twice live payload bytes;
      * the append tail contributes only one bounded extent beyond that ratio.
      */
     fragmentedExtentId(): Promise<number | undefined>;
+    /** Actual bytes of every checkpoint-known extent, including the held tail handle. */
+    physicalByteLengths(): Promise<Map<number, number>>;
     /** Live-byte accounting for reclaimed payloads; reports extents now safe to delete. */
     release(placements: Iterable<Placement>): number[];
     /** Removes a drained extent's file and forgets it. Idempotent. */
@@ -66,4 +98,6 @@ export declare class ExtentPool {
     close(): void;
 }
 export declare function extentPath(id: number): string[];
-//# sourceMappingURL=extents.d.ts.map
+export declare function validPlacement(placement: Placement): boolean;
+export declare function assertValidPlacement(placement: Placement): void;
+export declare function assertValidExtentMeta(meta: ExtentMeta): void;