@minnowdb/core 0.3.0 → 0.4.0

package/dist/engine/vector.js

@@ -1,3 +1,5 @@
+import { dateMilliseconds } from "../date-value.js";
+import { MAX_TEMP_RUN_BATCH_BYTES, MAX_TEMP_RUN_PAGE_BYTES, MAX_TEMP_RUN_PAGES_PER_BATCH, } from "../storage/types.js";
 import { cachedListMembership, childExpressions, distinctFromComparison, nullOrder, isScalarFunctionName, likeMatches, orderOutputName, parseQuantified, quantifiedComparison, scalarFunctionValue, } from "./query.js";
 import { jsonValueOf } from "./sql-json.js";
 import { bm25DocumentScore, cachedQueryTerms, FtsStatsAccumulator, fullTermsMask, renderDocumentValue, termFrequencies, termsMask, tokenize, } from "./fts.js";
@@ -27,7 +29,17 @@ const SPILL_PAGE_MAGIC = 0x5350494c;
 const SPILL_PAGE_HEADER_BYTES = 8;
 const vectorTextEncoder = new TextEncoder();
 /** Pages per batched spill write: bounds the transient encoded copies a batch holds. */
-const SPILL_WRITE_BATCH_PAGES = 8;
+const SPILL_WRITE_BATCH_PAGES = Math.min(8, MAX_TEMP_RUN_PAGES_PER_BATCH);
+function boundedSpillPageRows(value) {
+    const rows = value ?? DEFAULT_BATCH_ROWS;
+    if (!Number.isSafeInteger(rows) || rows <= 0) {
+        throw new RangeError("Query spill page rows must be a positive whole number");
+    }
+    // Page rows are a tuning target, not a request to buffer an unbounded number of rows. Larger
+    // pages do not improve the block-store round trip because byte-bounded batches provide that
+    // amortization independently.
+    return Math.min(rows, DEFAULT_BATCH_ROWS);
+}
 /** Writes already-materialized pages through the batch method when the store offers one. */
 async function writeSpillPages(store, pages) {
     const batched = store.putPages?.bind(store);
@@ -37,9 +49,43 @@ async function writeSpillPages(store, pages) {
         }
         return;
     }
-    for (let start = 0; start < pages.length; start += SPILL_WRITE_BATCH_PAGES) {
-        await batched(pages.slice(start, start + SPILL_WRITE_BATCH_PAGES));
+    let batch = [];
+    let batchBytes = 0;
+    for (const page of pages) {
+        if (page.bytes.byteLength > MAX_TEMP_RUN_PAGE_BYTES) {
+            throw new RangeError("Query spill page exceeds the storage limit");
+        }
+        if (batch.length > 0 &&
+            (batch.length === SPILL_WRITE_BATCH_PAGES ||
+                batchBytes + page.bytes.byteLength > MAX_TEMP_RUN_BATCH_BYTES)) {
+            await batched(batch);
+            batch = [];
+            batchBytes = 0;
+        }
+        batch.push(page);
+        batchBytes += page.bytes.byteLength;
     }
+    if (batch.length > 0)
+        await batched(batch);
+}
+async function writeSpillRowPages(store, ownerId, runId, startPageIndex, columns, rows, pageRows) {
+    let pending = [];
+    let pendingBytes = 0;
+    let pageCount = 0;
+    for (const bytes of encodeSpillRowPages(columns, rows, pageRows)) {
+        if (pending.length > 0 &&
+            (pending.length === SPILL_WRITE_BATCH_PAGES ||
+                pendingBytes + bytes.byteLength > MAX_TEMP_RUN_BATCH_BYTES)) {
+            await writeSpillPages(store, pending);
+            pending = [];
+            pendingBytes = 0;
+        }
+        pending.push({ ownerId, runId, pageIndex: startPageIndex + pageCount, bytes });
+        pendingBytes += bytes.byteLength;
+        pageCount += 1;
+    }
+    await writeSpillPages(store, pending);
+    return pageCount;
 }
 export function createColumnarTable(name, columns, uniqueKey) {
     const first = columns.values().next().value;
@@ -135,6 +181,23 @@ export function prepareVectorQuery(plan, inputTables, options = {}) {
                     executionMemory.close();
                 }
             },
+            async executeBatches(executionOptions, consume) {
+                if (closed)
+                    throw new Error("Prepared vector query is closed");
+                if (bound.grouped || bound.orderBy.length > 0 || bound.joins.length > 0) {
+                    throw new TypeError("Native query batching requires an ungrouped, unordered, single-source plan");
+                }
+                if (!Number.isSafeInteger(executionOptions.batchRows) || executionOptions.batchRows <= 0) {
+                    throw new RangeError("Query batch rows must be a positive whole number");
+                }
+                const executionMemory = retainedMemory.createChild();
+                try {
+                    return await executeBoundPlanBatches(bound, executionMemory, executionOptions, consume);
+                }
+                finally {
+                    executionMemory.close();
+                }
+            },
             close() {
                 if (closed)
                     return;
@@ -198,7 +261,7 @@ function createVector(input) {
             continue;
         const numericValue = input.type === "datetime"
             ? value instanceof Date
-                ? value.getTime()
+                ? dateMilliseconds(value)
                 : Number.NaN
             : typeof value === "number"
                 ? value
@@ -1202,15 +1265,61 @@ async function executeBoundPlanAsync(plan, memory, options) {
     const rows = plan.grouped ? finishGroups(plan, groups.values(), memory) : output.finish();
     return finishResult(plan, rows, memory);
 }
+/**
+ * Pull-driven scan execution for cursor-safe plans. A result page owns a child memory context
+ * that closes after the consumer settles, so both modeled memory and live row objects are
+ * bounded by one page. OFFSET/LIMIT are applied once across the whole scan, never per page.
+ */
+async function executeBoundPlanBatches(plan, memory, options, consume) {
+    const columns = plan.wildcard ? wildcardColumnNames(plan) : plan.select.map((item) => item.alias);
+    const { limit, offset, ...unbounded } = plan;
+    const scanPlan = unbounded;
+    let skipped = 0;
+    let emitted = 0;
+    const scanRows = scanPlan.sourceTables[scanPlan.scanSource]?.rowCount ?? 0;
+    const step = Math.min(DEFAULT_BATCH_ROWS, options.batchRows);
+    for (let start = 0; start < scanRows && (limit === undefined || emitted < limit);) {
+        options.signal?.throwIfAborted();
+        let length = Math.min(step, scanRows - start);
+        const loaded = options.loadScanWindow?.(start, length);
+        const residentEnd = typeof loaded === "number" || loaded === undefined ? loaded : await loaded;
+        options.signal?.throwIfAborted();
+        if (typeof residentEnd === "number" && residentEnd > start) {
+            length = Math.min(length, residentEnd - start);
+        }
+        const pageMemory = memory.createChild();
+        try {
+            const groups = new GroupAccumulator(scanPlan, pageMemory);
+            const output = new ResultSink(scanPlan, pageMemory, options.loadScanWindow === undefined);
+            runScanBatch(scanPlan, start, length, groups, output, pageMemory);
+            let rows = output.finish();
+            const remainingOffset = Math.max(0, (offset ?? 0) - skipped);
+            if (remainingOffset > 0) {
+                const remove = Math.min(remainingOffset, rows.length);
+                rows = rows.slice(remove);
+                skipped += remove;
+            }
+            if (limit !== undefined && rows.length > limit - emitted) {
+                rows = rows.slice(0, limit - emitted);
+            }
+            if (rows.length > 0) {
+                emitted += rows.length;
+                await consume({ columns: [...columns], rows });
+            }
+        }
+        finally {
+            pageMemory.close();
+        }
+        start += length;
+    }
+    return columns;
+}
 function createSpillOwnerId() {
     return `query-${globalThis.crypto.randomUUID()}`;
 }
 async function executeBoundPlanWithSortSpill(plan, memory, options) {
     const store = required(options.spillStore, "Query spill store is missing");
-    const pageRows = options.spillPageRows ?? DEFAULT_BATCH_ROWS;
-    if (!Number.isSafeInteger(pageRows) || pageRows <= 0) {
-        throw new RangeError("Query spill page rows must be a positive whole number");
-    }
+    const pageRows = boundedSpillPageRows(options.spillPageRows);
     const columns = plan.wildcard ? wildcardColumnNames(plan) : plan.select.map((item) => item.alias);
     const ownerId = createSpillOwnerId();
     const runs = [];
@@ -1251,8 +1360,8 @@ async function executeBoundPlanWithSortSpill(plan, memory, options) {
                             ordering.release();
                         }
                         const runId = `run-${String(runSequence++)}`;
-                        await store.putPage(ownerId, runId, 0, encodeSpillRows(columns, rows));
-                        runs.push({ id: runId, pageCount: 1 });
+                        const pageCount = await writeSpillRowPages(store, ownerId, runId, 0, columns, rows, pageRows);
+                        runs.push({ id: runId, pageCount });
                     }
                     finally {
                         outputMemory.close();
@@ -1307,10 +1416,7 @@ async function executeBoundPlanWithSortSpill(plan, memory, options) {
 }
 async function executeBoundPlanWithHashSpill(plan, memory, options) {
     const store = required(options.spillStore, "Query spill store is missing");
-    const pageRows = options.spillPageRows ?? DEFAULT_BATCH_ROWS;
-    if (!Number.isSafeInteger(pageRows) || pageRows <= 0) {
-        throw new RangeError("Query spill page rows must be a positive whole number");
-    }
+    const pageRows = boundedSpillPageRows(options.spillPageRows);
     const partitionCount = 64;
     const columns = plan.select.map((item) => item.alias);
     const groupColumnNames = plan.groupBy.map((_, index) => `g${String(index)}`);
@@ -1382,15 +1488,26 @@ async function executeBoundPlanWithHashSpill(plan, memory, options) {
                     }
                 });
                 const flush = [];
+                let flushBytes = 0;
                 for (const [partition, rows] of partitionBuffers) {
-                    const pageIndex = partitionPages[partition] ?? 0;
-                    flush.push({
-                        ownerId,
-                        runId: `partition-${String(partition)}`,
-                        pageIndex,
-                        bytes: encodeSpillRows(spillColumns, rows),
-                    });
-                    partitionPages[partition] = pageIndex + 1;
+                    for (const bytes of encodeSpillRowPages(spillColumns, rows, pageRows)) {
+                        if (flush.length > 0 &&
+                            (flush.length === SPILL_WRITE_BATCH_PAGES ||
+                                flushBytes + bytes.byteLength > MAX_TEMP_RUN_BATCH_BYTES)) {
+                            await writeSpillPages(store, flush);
+                            flush.length = 0;
+                            flushBytes = 0;
+                        }
+                        const pageIndex = partitionPages[partition] ?? 0;
+                        flush.push({
+                            ownerId,
+                            runId: `partition-${String(partition)}`,
+                            pageIndex,
+                            bytes,
+                        });
+                        flushBytes += bytes.byteLength;
+                        partitionPages[partition] = pageIndex + 1;
+                    }
                 }
                 await writeSpillPages(store, flush);
             }
@@ -1437,23 +1554,8 @@ async function executeBoundPlanWithHashSpill(plan, memory, options) {
                     }
                 }
                 const runId = `group-${String(runSequence++)}`;
-                let outputPage = 0;
-                let pending = [];
-                for (let start = 0; start < rows.length; start += pageRows) {
-                    pending.push({
-                        ownerId,
-                        runId,
-                        pageIndex: outputPage,
-                        bytes: encodeSpillRows(columns, rows.slice(start, start + pageRows)),
-                    });
-                    outputPage += 1;
-                    if (pending.length >= SPILL_WRITE_BATCH_PAGES) {
-                        await writeSpillPages(store, pending);
-                        pending = [];
-                    }
-                }
-                await writeSpillPages(store, pending);
-                runs.push({ id: runId, pageCount: outputPage });
+                const pageCount = await writeSpillRowPages(store, ownerId, runId, 0, columns, rows, pageRows);
+                runs.push({ id: runId, pageCount });
             }
             finally {
                 partitionMemory.close();
@@ -1589,11 +1691,10 @@ async function mergeSpillRuns(store, ownerId, left, right, outputId, columns, or
     const flush = async () => {
         if (outputPage.length === 0)
             return;
-        await store.putPage(ownerId, outputId, pageIndex, encodeSpillRows(columns, outputPage));
+        pageIndex += await writeSpillRowPages(store, ownerId, outputId, pageIndex, columns, outputPage, pageRows);
         outputPage = [];
         outputMemory.close();
         outputMemory = mergeMemory.createChild();
-        pageIndex += 1;
     };
     try {
         let leftRow = await leftReader.next();
@@ -1665,19 +1766,63 @@ function compareOrderedRows(left, right, orderBy) {
     }
     return 0;
 }
-function encodeSpillRows(columns, rows) {
-    const encoded = rows.map((row) => columns.map((column) => encodeSpillValue(row[column] ?? null)));
-    const payload = vectorTextEncoder.encode(JSON.stringify(encoded));
-    const modeledBytes = rows.reduce((total, row) => safeMemorySum(total, queryRowPayloadBytes(row), "Spill page rows"), 0);
-    if (modeledBytes > 0xffffffff)
-        throw new RangeError("Spill page modeled bytes exceed uint32");
-    const bytes = new Uint8Array(SPILL_PAGE_HEADER_BYTES + payload.byteLength);
+function encodeSpillRow(columns, row) {
+    return {
+        bytes: vectorTextEncoder.encode(JSON.stringify(columns.map((column) => encodeSpillValue(row[column] ?? null)))),
+        modeledBytes: queryRowPayloadBytes(row),
+    };
+}
+function buildSpillPage(rows, byteLength, modeledBytes) {
+    const bytes = new Uint8Array(byteLength);
     const header = new DataView(bytes.buffer);
     header.setUint32(0, SPILL_PAGE_MAGIC, true);
     header.setUint32(4, modeledBytes, true);
-    bytes.set(payload, SPILL_PAGE_HEADER_BYTES);
+    let offset = SPILL_PAGE_HEADER_BYTES;
+    bytes[offset++] = 0x5b;
+    for (let index = 0; index < rows.length; index += 1) {
+        if (index > 0)
+            bytes[offset++] = 0x2c;
+        const row = required(rows[index], "Encoded spill row is missing");
+        bytes.set(row.bytes, offset);
+        offset += row.bytes.byteLength;
+    }
+    bytes[offset] = 0x5d;
     return bytes;
 }
+/** Encodes pages incrementally and splits on both the requested row target and the hard byte cap. */
+function* encodeSpillRowPages(columns, rows, pageRows) {
+    let pending = [];
+    let byteLength = SPILL_PAGE_HEADER_BYTES + 2;
+    let modeledBytes = 0;
+    const flush = () => {
+        const page = buildSpillPage(pending, byteLength, modeledBytes);
+        pending = [];
+        byteLength = SPILL_PAGE_HEADER_BYTES + 2;
+        modeledBytes = 0;
+        return page;
+    };
+    for (const row of rows) {
+        const encoded = encodeSpillRow(columns, row);
+        const separatorBytes = pending.length === 0 ? 0 : 1;
+        const nextByteLength = byteLength + separatorBytes + encoded.bytes.byteLength;
+        const nextModeledBytes = modeledBytes + encoded.modeledBytes;
+        if (pending.length > 0 &&
+            (pending.length === pageRows ||
+                nextByteLength > MAX_TEMP_RUN_PAGE_BYTES ||
+                nextModeledBytes > 0xffffffff)) {
+            yield flush();
+        }
+        const addedSeparatorBytes = pending.length === 0 ? 0 : 1;
+        byteLength += addedSeparatorBytes + encoded.bytes.byteLength;
+        modeledBytes += encoded.modeledBytes;
+        if (byteLength > MAX_TEMP_RUN_PAGE_BYTES || modeledBytes > 0xffffffff) {
+            throw new RangeError("One query result row exceeds the spill page storage limit");
+        }
+        pending.push(encoded);
+    }
+    if (pending.length > 0)
+        yield flush();
+}
 function decodeSpillRows(columns, bytes) {
     spillRowsModeledBytes(bytes);
     const value = JSON.parse(new TextDecoder().decode(bytes.subarray(SPILL_PAGE_HEADER_BYTES)));
@@ -1708,7 +1853,7 @@ function encodeSpillValue(value) {
         return [2, Object.is(value, -0) ? "-0" : String(value)];
     if (typeof value === "string")
         return [3, value];
-    return [4, value.getTime()];
+    return [4, dateMilliseconds(value)];
 }
 function decodeSpillValue(value) {
     if (!Array.isArray(value))
@@ -2851,7 +2996,7 @@ class ResultSink {
                     : typeof first === "number"
                         ? first
                         : first instanceof Date
-                            ? first.getTime()
+                            ? dateMilliseconds(first)
                             : undefined;
     }
     /**
@@ -3404,7 +3549,7 @@ function createGroupState(groupValues, plan, memory) {
  * string a row genuinely holds.
  */
 function distinctKey(value) {
-    return value instanceof Date ? `d${String(value.getTime())}` : value;
+    return value instanceof Date ? `\0d${String(dateMilliseconds(value))}` : value;
 }
 /**
  * Whether this value is the first of its kind for the slot, folding it into the set when it is.
@@ -3799,7 +3944,7 @@ function detectPrimitiveComparison(predicate) {
             return { source: column.source, vector, operator, value };
         }
         if (vector.kind === "datetime" && value instanceof Date) {
-            return { source: column.source, vector, operator, value: value.getTime() };
+            return { source: column.source, vector, operator, value: dateMilliseconds(value) };
         }
         if (vector.kind === "boolean" &&
             typeof value === "boolean" &&
@@ -3847,7 +3992,7 @@ function detectPrimitiveInList(predicate) {
         else {
             if (!(value instanceof Date))
                 return undefined;
-            members.add(value.getTime());
+            members.add(dateMilliseconds(value));
         }
     }
     if (members.size === 0)
@@ -4361,7 +4506,7 @@ function comparisonValue(operator, leftValue, rightValue) {
     return comparison <= 0;
 }
 function comparable(value) {
-    return value instanceof Date ? value.getTime() : value;
+    return value instanceof Date ? dateMilliseconds(value) : value;
 }
 function groupKey(value) {
     const comparableValue = comparable(value);
@@ -4411,8 +4556,8 @@ function compareValues(left, right) {
     const exact = exactNumericCompare(left, right);
     if (exact !== undefined)
         return exact;
-    const a = left instanceof Date ? left.getTime() : left;
-    const b = right instanceof Date ? right.getTime() : right;
+    const a = left instanceof Date ? dateMilliseconds(left) : left;
+    const b = right instanceof Date ? dateMilliseconds(right) : right;
     if (a === b)
         return 0;
     if (a === null || a === undefined)
@@ -4492,4 +4637,3 @@ function safeMemoryProduct(left, right, label) {
     }
     return product;
 }
-//# sourceMappingURL=vector.js.map

package/dist/engine/worker-host.d.ts

@@ -21,7 +21,7 @@ export type StoreDescriptor = {
     durability?: "relaxed" | "strict";
 };
 /** The cloneable subset of MinnowDatabaseOptions; function-valued seams stay worker-side. */
-export type WireDatabaseOptions = Pick<MinnowDatabaseOptions, "compression" | "rowsPerBlock" | "maxCommitRetries" | "spillOwnerLeaseMs" | "bufferPoolBytes">;
+export type WireDatabaseOptions = Pick<MinnowDatabaseOptions, "compression" | "targetBlockBytes" | "rowsPerBlock" | "maxCommitRetries" | "spillOwnerLeaseMs" | "transactionOwnerLeaseMs" | "transactionIdleTimeoutMs" | "bufferPoolBytes" | "executionMemoryBudgetBytes" | "autoCollectDebtLimitCommits" | "autoCollect" | "autoCompact">;
 export interface DatabaseInitPayload {
     store: StoreDescriptor;
     options?: WireDatabaseOptions;
@@ -40,6 +40,11 @@ export interface RpcScope {
 export interface ExposeDatabaseOptions {
     /** Called when the client sends its dispose frame, after every handle has been closed. */
     onDispose?: () => void | Promise<void>;
+    /**
+     * Rolls back a worker write handle after this long without a handle RPC. Match the constructed
+     * database's `transactionIdleTimeoutMs` when it is customized. Defaults to 30 seconds.
+     */
+    writeHandleIdleTimeoutMs?: number;
     /**
      * Called when the main thread reports its page visibility. The stock entry forwards this to
      * the store's `setForeground` when it has one — the OPFS store uses it to keep leadership
@@ -51,6 +56,8 @@ export interface AttachDatabaseWorkerOptions {
     /** Test seam for environments without a global IndexedDB (e.g. fake-indexeddb). */
     indexedDB?: IDBFactory;
 }
+/** Hard per-connection ceiling for resident worker resources abandoned by a client. */
+export declare const MAX_WORKER_HANDLES_PER_CONNECTION = 256;
 /**
  * Answers client RPC frames against a database you constructed yourself. Use this in a custom
  * worker entry when you need non-cloneable construction options (a custom store, `now`,
@@ -62,4 +69,3 @@ export declare function exposeDatabase(database: MinnowDatabase, scope: RpcScope
  * constructs the database, and exposes it. The store is closed when the client disposes.
  */
 export declare function attachDatabaseWorker(scope: RpcScope, options?: AttachDatabaseWorkerOptions): void;
-//# sourceMappingURL=worker-host.d.ts.map