@minnowdb/core 0.2.0 → 0.3.0

package/dist/storage/opfs/leader.js

@@ -1,7 +1,8 @@
-import { collectFtsCandidates, } from "../types.js";
+import { activePostingStorageColumnIds, collectFtsCandidates, collectFtsPostings, } from "../types.js";
+import { validateSnapshotCatalog, } from "../snapshot.js";
 import { RecordCore, validateBeginTransactionInput, validateId, validateTempRunPage, validateTempRunPageIdentity, } from "../toolkit/record-core.js";
 import { encodeSegment, decodeSegment } from "./files.js";
-import { decodeChunk, decodeSyncCheckpoint, encodeChunk, encodeSyncCheckpoint, } from "../toolkit/wire.js";
+import { decodeChunk, decodePostingChunk, decodeSyncCheckpoint, encodePostingChunk, encodeSyncCheckpoint, } from "../toolkit/wire.js";
 import { WalWriter, replayWalFrames } from "../toolkit/wal.js";
 import { ExtentPool } from "../toolkit/extents.js";
 /** Checkpoint when the WAL passes either bound; both trade replay time against pause time. */
@@ -9,6 +10,8 @@ const CHECKPOINT_WAL_BYTES = 4 * 1024 * 1024;
 const CHECKPOINT_ENTRIES = 1024;
 /** Decoded full-text base chunks kept in memory; least recently loaded goes first. */
 const FTS_CHUNK_CACHE_SIZE = 64;
+/** Extent reads in flight for a range or ordered postings scan. */
+const FTS_CHUNK_READ_CONCURRENCY = 8;
 /**
  * The leader: the one instance holding the database's write-ahead log handle, and with it
  * every other handle — checkpoint slots, the tail extent, a small cache of sealed extents.
@@ -32,6 +35,7 @@ export class OpfsLeader {
     #core;
     #blockIndex = new Map();
     #ftsBases = new Map();
+    #ftsBuilds = new Map();
     #ftsChunkCache = new Map();
     #extents;
     #wal;
@@ -98,6 +102,7 @@ export class OpfsLeader {
         });
         this.#blockIndex.clear();
         this.#ftsBases.clear();
+        this.#ftsBuilds.clear();
         this.#ftsChunkCache.clear();
         this.#extents?.close();
         this.#extents = await ExtentPool.open(this.#tree, checkpoint?.extents);
@@ -111,6 +116,8 @@ export class OpfsLeader {
                 this.#blockIndex.set(id, placement);
             for (const [key, pointer] of checkpoint.ftsBases)
                 this.#ftsBases.set(key, pointer);
+            for (const [key, pointer] of checkpoint.ftsBuilds ?? [])
+                this.#ftsBuilds.set(key, pointer);
             this.#seq = checkpoint.lastSeq;
         }
         const newest = this.#slots[this.#newestSlot];
@@ -141,6 +148,16 @@ export class OpfsLeader {
         this.#wal = new WalWriter(this.#walHandle, endOffset);
         this.#entriesSinceCheckpoint = applied;
         this.#poisoned = false;
+        // A staged postings generation has no reader-visible pointer and no builder can prove it
+        // survived a leader death. Reclaim it now rather than retain unreachable extents. A
+        // still-live caller receives "build changed", marks its catalog record invalid, and a later
+        // query restarts from a fresh snapshot. The next ordinary checkpoint persists the
+        // reclamation; until then another crash simply replays and reclaims the same unpublished
+        // generation.
+        if (this.#ftsBuilds.size > 0) {
+            this.#pool.release([...this.#ftsBuilds.values()].flatMap((build) => build.chunks));
+            this.#ftsBuilds.clear();
+        }
         // Extents fully drained before the crash but never deleted: finish the job.
         for (const id of this.#pool.release([]))
             await this.#pool.deleteExtent(id);
@@ -240,6 +257,13 @@ export class OpfsLeader {
                         throw new Error(`OPFS full-text chunk moved before relocation: ${move.key}`);
                     }
                 }
+                for (const move of body.ftsBuildChunks ?? []) {
+                    const build = this.#ftsBuilds.get(move.key);
+                    if (build?.buildId !== move.buildId ||
+                        !samePlacement(build.chunks[move.ordinal], move.from)) {
+                        throw new Error(`OPFS staged postings chunk moved before relocation: ${move.key}`);
+                    }
+                }
                 for (const move of body.blocks)
                     this.#blockIndex.set(move.id, move.placement);
                 for (const move of body.ftsChunks) {
@@ -249,16 +273,53 @@ export class OpfsLeader {
                     pointer.chunks[move.ordinal] = move.placement;
                     this.#dropFtsChunkCache(move.key);
                 }
+                for (const move of body.ftsBuildChunks ?? []) {
+                    const build = this.#ftsBuilds.get(move.key);
+                    if (build === undefined) {
+                        throw new Error(`OPFS staged postings build is missing: ${move.key}`);
+                    }
+                    build.chunks[move.ordinal] = move.placement;
+                }
                 this.#pool.release([
                     ...body.blocks.map((move) => move.from),
                     ...body.ftsChunks.map((move) => move.from),
+                    ...(body.ftsBuildChunks ?? []).map((move) => move.from),
                 ]);
                 return undefined;
             }
             case "addTable":
                 return this.#core.addTable(body.record);
-            case "updateTable":
-                return this.#core.updateTable(body.id, body.expectedRevision, body.update);
+            case "updateTable": {
+                const before = this.#core.getTable(body.id);
+                const updated = this.#core.updateTable(body.id, body.expectedRevision, body.update);
+                if (before !== undefined) {
+                    const beforeStorageIds = new Set([
+                        ...Object.keys(before.ftsColumns ?? {}),
+                        ...Object.values(before.secondaryIndexes ?? {}).map((index) => index.storageColumnId),
+                    ]);
+                    const retainedStorageIds = new Set([
+                        ...Object.keys(updated.ftsColumns ?? {}),
+                        ...Object.values(updated.secondaryIndexes ?? {}).map((index) => index.storageColumnId),
+                    ]);
+                    for (const storageColumnId of beforeStorageIds) {
+                        if (retainedStorageIds.has(storageColumnId))
+                            continue;
+                        const key = `${before.id}/${storageColumnId}`;
+                        const pointer = this.#ftsBases.get(key);
+                        if (pointer !== undefined) {
+                            this.#ftsBases.delete(key);
+                            this.#dropFtsChunkCache(key);
+                            this.#pool.release(pointer.chunks);
+                        }
+                        const build = this.#ftsBuilds.get(key);
+                        if (build !== undefined) {
+                            this.#ftsBuilds.delete(key);
+                            this.#pool.release(build.chunks);
+                        }
+                    }
+                }
+                return updated;
+            }
             case "removeTable": {
                 this.#core.removeTable(body.id, body.expectedRevision);
                 const owned = `${body.id}/`;
@@ -269,6 +330,28 @@ export class OpfsLeader {
                     this.#dropFtsChunkCache(key);
                     this.#pool.release(pointer.chunks);
                 }
+                for (const [key, build] of [...this.#ftsBuilds]) {
+                    if (!key.startsWith(owned))
+                        continue;
+                    this.#ftsBuilds.delete(key);
+                    this.#pool.release(build.chunks);
+                }
+                return undefined;
+            }
+            case "removeFtsColumn": {
+                const key = `${body.tableId}/${body.columnId}`;
+                const pointer = this.#ftsBases.get(key);
+                if (pointer !== undefined) {
+                    this.#ftsBases.delete(key);
+                    this.#dropFtsChunkCache(key);
+                    this.#pool.release(pointer.chunks);
+                }
+                const build = this.#ftsBuilds.get(key);
+                if (build !== undefined) {
+                    this.#ftsBuilds.delete(key);
+                    this.#pool.release(build.chunks);
+                }
+                this.#core.removeFtsColumn(body.tableId, body.columnId);
                 return undefined;
             }
             case "addSegment":
@@ -347,6 +430,11 @@ export class OpfsLeader {
                 return this.#core.removeTempOwner(body.ownerId);
             case "writeFtsBase": {
                 const key = `${body.tableId}/${body.columnId}`;
+                const build = this.#ftsBuilds.get(key);
+                if (build !== undefined) {
+                    this.#ftsBuilds.delete(key);
+                    this.#pool.release(build.chunks);
+                }
                 const previous = this.#ftsBases.get(key);
                 if (previous !== undefined) {
                     this.#pool.release(previous.chunks);
@@ -356,6 +444,67 @@ export class OpfsLeader {
                 this.#core.pruneFtsDeltas(body.tableId, body.columnId, body.pointer.coversVersion);
                 return undefined;
             }
+            case "beginFtsBaseBuild": {
+                const table = this.#core.getTable(body.tableId);
+                if (table === undefined || !activePostingStorageColumnIds(table).has(body.columnId)) {
+                    throw new Error(`Postings index is no longer active: ${body.tableId}/${body.columnId}`);
+                }
+                const key = `${body.tableId}/${body.columnId}`;
+                const previous = this.#ftsBuilds.get(key);
+                if (previous !== undefined)
+                    this.#pool.release(previous.chunks);
+                this.#ftsBuilds.set(key, { buildId: body.buildId, chunks: [], chunkBounds: [] });
+                return undefined;
+            }
+            case "writeFtsBaseBuildChunk": {
+                const key = `${body.tableId}/${body.columnId}`;
+                const build = this.#ftsBuilds.get(key);
+                if (build?.buildId !== body.buildId) {
+                    throw new Error(`Full-text base build changed: ${body.buildId}`);
+                }
+                if (body.ordinal !== build.chunks.length) {
+                    throw new Error(`Full-text base chunk is out of order: ${String(body.ordinal)}`);
+                }
+                build.chunks.push(body.placement);
+                build.chunkBounds.push(body.bounds);
+                return undefined;
+            }
+            case "finishFtsBaseBuild": {
+                const key = `${body.tableId}/${body.columnId}`;
+                const build = this.#ftsBuilds.get(key);
+                if (build?.buildId !== body.buildId || build.chunks.length !== body.input.chunkCount) {
+                    throw new Error(`Full-text base build is incomplete: ${body.buildId}`);
+                }
+                const table = this.#core.getTable(body.tableId);
+                if (table === undefined || !activePostingStorageColumnIds(table).has(body.columnId)) {
+                    this.#ftsBuilds.delete(key);
+                    this.#pool.release(build.chunks);
+                    return undefined;
+                }
+                const previous = this.#ftsBases.get(key);
+                if (previous !== undefined) {
+                    this.#pool.release(previous.chunks);
+                    this.#dropFtsChunkCache(key);
+                }
+                this.#ftsBases.set(key, {
+                    coversVersion: body.input.coversVersion,
+                    totalTokens: body.input.totalTokens,
+                    chunks: build.chunks,
+                    chunkBounds: build.chunkBounds,
+                });
+                this.#ftsBuilds.delete(key);
+                this.#core.pruneFtsDeltas(body.tableId, body.columnId, body.input.coversVersion);
+                return undefined;
+            }
+            case "abortFtsBaseBuild": {
+                const key = `${body.tableId}/${body.columnId}`;
+                const build = this.#ftsBuilds.get(key);
+                if (build?.buildId === body.buildId) {
+                    this.#ftsBuilds.delete(key);
+                    this.#pool.release(build.chunks);
+                }
+                return undefined;
+            }
             case "importSnapshot": {
                 if (this.#core.getCurrentManifestVersion() !== null || this.#core.listTables().length > 0) {
                     throw new Error("This store already holds a database");
@@ -395,6 +544,7 @@ export class OpfsLeader {
             core: this.#core.dump(),
             blockIndex: [...this.#blockIndex.entries()],
             ftsBases: [...this.#ftsBases.entries()],
+            ftsBuilds: [...this.#ftsBuilds.entries()],
             extents: this.#pool.meta(),
         };
         const bytes = encodeSyncCheckpoint(state);
@@ -423,11 +573,13 @@ export class OpfsLeader {
                 .filter(([, placement]) => placement.extent === extent)
                 .map(([id, from]) => ({ id, from }));
             const ftsSources = [...this.#ftsBases.entries()].flatMap(([key, pointer]) => pointer.chunks.flatMap((from, ordinal) => from.extent === extent ? [{ key, ordinal, from }] : []));
-            if (blockSources.length === 0 && ftsSources.length === 0) {
+            const ftsBuildSources = [...this.#ftsBuilds.entries()].flatMap(([key, build]) => build.chunks.flatMap((from, ordinal) => from.extent === extent ? [{ key, buildId: build.buildId, ordinal, from }] : []));
+            if (blockSources.length === 0 && ftsSources.length === 0 && ftsBuildSources.length === 0) {
                 throw new Error(`OPFS extent ${String(extent)} has live bytes but no indexed payload`);
             }
             const blocks = [];
             const ftsChunks = [];
+            const ftsBuildChunks = [];
             const appended = [];
             try {
                 for (const source of blockSources) {
@@ -440,7 +592,12 @@ export class OpfsLeader {
                     appended.push(placement);
                     ftsChunks.push({ ...source, placement });
                 }
-                this.#logged({ op: "relocatePayloads", blocks, ftsChunks });
+                for (const source of ftsBuildSources) {
+                    const placement = await this.#pool.append(await this.#pool.read(source.from), this.#strict);
+                    appended.push(placement);
+                    ftsBuildChunks.push({ ...source, placement });
+                }
+                this.#logged({ op: "relocatePayloads", blocks, ftsChunks, ftsBuildChunks });
             }
             catch (error) {
                 this.#pool.release(appended);
@@ -574,6 +731,10 @@ export class OpfsLeader {
         await this.#healthy();
         return this.#core.listSegments(tableId);
     }
+    async listSegmentPage(afterId, limit) {
+        await this.#healthy();
+        return this.#core.listSegmentPage(afterId, limit);
+    }
     async getExistingUniqueKeys(tableId, keyTokens) {
         await this.#healthy();
         return this.#core.getExistingUniqueKeys(tableId, keyTokens);
@@ -657,7 +818,21 @@ export class OpfsLeader {
         await this.#run(() => this.#logged({ op: "addTable", record }));
     }
     async updateTable(id, expectedRevision, update) {
-        return this.#run(() => this.#logged({ op: "updateTable", id, expectedRevision, update }));
+        return this.#run(async () => {
+            const updated = this.#logged({
+                op: "updateTable",
+                id,
+                expectedRevision,
+                update,
+            });
+            if (update.columns !== undefined) {
+                // DROP COLUMN may have released full-text chunks. Reclaim their extents before this
+                // mutation resolves so repeated schema evolution cannot grow the live process's disk.
+                await this.#deleteDrainedExtents();
+                await this.#compactFragmentedExtents();
+            }
+            return updated;
+        });
     }
     async removeTable(id, expectedRevision) {
         await this.#run(async () => {
@@ -666,6 +841,13 @@ export class OpfsLeader {
             await this.#compactFragmentedExtents();
         });
     }
+    async removeFtsColumn(tableId, columnId) {
+        await this.#run(async () => {
+            this.#logged({ op: "removeFtsColumn", tableId, columnId });
+            await this.#deleteDrainedExtents();
+            await this.#compactFragmentedExtents();
+        });
+    }
     async addSegment(record) {
         await this.#run(() => this.#logged({ op: "addSegment", record }));
     }
@@ -884,21 +1066,95 @@ export class OpfsLeader {
             await this.#compactFragmentedExtents();
         });
     }
+    async beginFtsBaseBuild(tableId, columnId, buildId) {
+        await this.#run(() => {
+            const table = this.#core.getTable(tableId);
+            if (table === undefined || !activePostingStorageColumnIds(table).has(columnId)) {
+                throw new Error(`Postings index is no longer active: ${tableId}/${columnId}`);
+            }
+            this.#logged({ op: "beginFtsBaseBuild", tableId, columnId, buildId });
+        });
+    }
+    async writeFtsBaseBuildChunk(tableId, columnId, buildId, ordinal, chunk) {
+        await this.#run(async () => {
+            const build = this.#ftsBuilds.get(`${tableId}/${columnId}`);
+            if (build?.buildId !== buildId)
+                throw new Error(`Full-text base build changed: ${buildId}`);
+            if (ordinal !== build.chunks.length) {
+                throw new Error(`Full-text base chunk is out of order: ${String(ordinal)}`);
+            }
+            const placement = await this.#pool.append(encodeFtsChunk(chunk), this.#strict);
+            this.#logged({
+                op: "writeFtsBaseBuildChunk",
+                tableId,
+                columnId,
+                buildId,
+                ordinal,
+                placement,
+                bounds: { first: chunk[0]?.term ?? "", last: chunk[chunk.length - 1]?.term ?? "" },
+            });
+        });
+    }
+    async finishFtsBaseBuild(tableId, columnId, buildId, input) {
+        await this.#run(async () => {
+            this.#logged({ op: "finishFtsBaseBuild", tableId, columnId, buildId, input });
+            await this.#deleteDrainedExtents();
+            await this.#compactFragmentedExtents();
+        });
+    }
+    async abortFtsBaseBuild(tableId, columnId, buildId) {
+        await this.#run(async () => {
+            this.#logged({ op: "abortFtsBaseBuild", tableId, columnId, buildId });
+            await this.#deleteDrainedExtents();
+        });
+    }
     async readFtsCandidates(tableId, columnId, terms, upToVersion) {
-        await this.#healthy();
-        const key = `${tableId}/${columnId}`;
-        const pointer = this.#ftsBases.get(key);
-        const coversVersion = pointer?.coversVersion ?? -1;
-        const deltas = this.#core.readFtsDeltas(tableId, columnId, coversVersion, upToVersion);
-        const baseChunks = pointer === undefined
-            ? []
-            : await Promise.all(selectFtsChunks(pointer.chunkBounds, terms).map((ordinal) => this.#loadFtsChunk(key, pointer, ordinal)));
-        return {
-            ...collectFtsCandidates([...baseChunks, ...deltas.chunkLists], terms),
-            deltaChunkCount: deltas.deltaChunkCount,
-            totalTokens: (pointer?.totalTokens ?? 0) + deltas.deltaTokens,
-            coversVersion,
-        };
+        return this.#run(async () => {
+            // Unlike immutable table blocks, derived-index extents are not protected by reader leases:
+            // DROP INDEX may reclaim them immediately. Hold the leader queue until every selected
+            // chunk is copied so a concurrent drop yields either the complete old base or no base,
+            // never a missing file halfway through a query.
+            const key = `${tableId}/${columnId}`;
+            const pointer = this.#ftsBases.get(key);
+            const coversVersion = pointer?.coversVersion ?? -1;
+            const deltas = this.#core.readFtsDeltas(tableId, columnId, coversVersion, upToVersion);
+            const baseChunks = pointer === undefined
+                ? []
+                : await this.#loadFtsChunks(key, pointer, selectFtsChunks(pointer.chunkBounds, terms));
+            return {
+                ...collectFtsCandidates([...baseChunks, ...deltas.chunkLists], terms),
+                deltaChunkCount: deltas.deltaChunkCount,
+                totalTokens: (pointer?.totalTokens ?? 0) + deltas.deltaTokens,
+                coversVersion,
+                hasBase: pointer !== undefined,
+            };
+        });
+    }
+    async readFtsPostings(tableId, columnId, upToVersion) {
+        return this.#run(async () => {
+            const key = `${tableId}/${columnId}`;
+            const pointer = this.#ftsBases.get(key);
+            const coversVersion = pointer?.coversVersion ?? -1;
+            const deltas = this.#core.readFtsDeltas(tableId, columnId, coversVersion, upToVersion);
+            const baseChunks = pointer === undefined
+                ? []
+                : await this.#loadFtsChunks(key, pointer, pointer.chunks.map((_, ordinal) => ordinal));
+            return {
+                postings: collectFtsPostings([...baseChunks, ...deltas.chunkLists]),
+                deltaChunkCount: deltas.deltaChunkCount,
+                coversVersion,
+                hasBase: pointer !== undefined,
+            };
+        });
+    }
+    async #loadFtsChunks(key, pointer, ordinals) {
+        const chunks = [];
+        for (let start = 0; start < ordinals.length; start += FTS_CHUNK_READ_CONCURRENCY) {
+            chunks.push(...(await Promise.all(ordinals
+                .slice(start, start + FTS_CHUNK_READ_CONCURRENCY)
+                .map((ordinal) => this.#loadFtsChunk(key, pointer, ordinal)))));
+        }
+        return chunks;
     }
     async #loadFtsChunk(key, pointer, ordinal) {
         const placement = pointer.chunks[ordinal];
@@ -944,6 +1200,7 @@ export class OpfsLeader {
         });
     }
     async importSnapshot(snapshot, options = {}) {
+        validateSnapshotCatalog(snapshot.tables);
         const totalBytes = snapshot.blocks.reduce((total, block) => total + block.bytes.byteLength, 0);
         options.onProgress?.({ phase: "blocks", writtenBytes: 0, totalBytes });
         await this.#run(async () => {
@@ -1052,12 +1309,15 @@ function placementsOf(entry) {
             return entry.blocks.map(({ placement }) => placement);
         case "writeFtsBase":
             return entry.pointer.chunks;
+        case "writeFtsBaseBuildChunk":
+            return [entry.placement];
         case "importSnapshot":
             return entry.blockPlacements.map(({ placement }) => placement);
         case "relocatePayloads":
             return [
                 ...entry.blocks.map(({ placement }) => placement),
                 ...entry.ftsChunks.map(({ placement }) => placement),
+                ...(entry.ftsBuildChunks ?? []).map(({ placement }) => placement),
             ];
         default:
             return [];
@@ -1067,10 +1327,12 @@ function samePlacement(left, right) {
     return (left?.extent === right.extent && left.offset === right.offset && left.length === right.length);
 }
 function encodeFtsChunk(chunk) {
-    return encodeChunk(chunk);
+    return encodePostingChunk(chunk);
 }
 function decodeFtsChunk(bytes) {
-    const chunk = decodeChunk(bytes);
+    // Generic JSON chunks were written by format-v2 releases before the compact postings codec.
+    // Keep reading them in place; the next rebuild naturally writes the compact representation.
+    const chunk = decodePostingChunk(bytes) ?? decodeChunk(bytes);
     if (chunk === undefined)
         throw new Error("Full-text base chunk is unreadable");
     return chunk;
@@ -1079,13 +1341,11 @@ function decodeFtsChunk(bytes) {
 function selectFtsChunks(chunkBounds, terms) {
     const ordinals = [];
     for (const [ordinal, bounds] of chunkBounds.entries()) {
-        const needed = terms.some(({ term, prefix }) => {
-            if (bounds.last < term)
-                return false;
-            if (prefix) {
-                return bounds.first <= `${term}￿` || bounds.first.startsWith(term);
-            }
-            return bounds.first <= term;
+        const needed = terms.some((query) => {
+            const lower = "term" in query ? query.term : query.lower;
+            const upper = "term" in query ? (query.prefix ? `${query.term}￿` : query.term) : query.upper;
+            return ((lower === undefined || lower <= bounds.last) &&
+                (upper === undefined || upper >= bounds.first));
         });
         if (needed)
             ordinals.push(ordinal);
@@ -1093,12 +1353,19 @@ function selectFtsChunks(chunkBounds, terms) {
     return ordinals;
 }
 function invalidateFtsColumns(record) {
-    if (record.ftsColumns === undefined)
-        return record;
-    const ftsColumns = Object.fromEntries(Object.entries(record.ftsColumns).map(([columnId, state]) => [
+    const ftsColumns = Object.fromEntries(Object.entries(record.ftsColumns ?? {}).map(([columnId, state]) => [
         columnId,
         { ...state, state: "invalid" },
     ]));
-    return { ...record, ftsColumns };
+    const secondaryIndexes = Object.fromEntries(Object.entries(record.secondaryIndexes ?? {}).map(([indexId, state]) => {
+        const { buildId: _abandonedBuild, ...invalid } = state;
+        void _abandonedBuild;
+        return [indexId, { ...invalid, state: "invalid" }];
+    }));
+    return {
+        ...record,
+        ...(Object.keys(ftsColumns).length === 0 ? {} : { ftsColumns }),
+        ...(Object.keys(secondaryIndexes).length === 0 ? {} : { secondaryIndexes }),
+    };
 }
 //# sourceMappingURL=leader.js.map