@minnowdb/core 0.2.1 → 0.4.0

package/dist/engine/client.js

@@ -1,7 +1,8 @@
-import { CompactionJobConflictError, GarbageCollectionJobConflictError, LeaseConflictError, SnapshotManifestMissingError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyConflictError, WriteConflictError, } from "../storage/index.js";
-import { parseRpcResponse, protocolVersion, } from "../worker-protocol/index.js";
+import { BlockReadBatchTooLargeError, CompactionBacklogError, CompactionJobConflictError, GarbageCollectionJobConflictError, IndexedDbSchemaUpgradeBlockedError, LeaseConflictError, LeaseExpiredError, LeaseOwnerConflictError, PostingBuildConflictError, SnapshotManifestMissingError, SnapshotImportConflictError, SchemaConflictError, StorageResourceLimitError, TableInUseError, TableRecordConflictError, TempOwnerConflictError, TransactionRecordConflictError, UniqueKeyBuildConflictError, UniqueKeyConflictError, UniqueIndexCoverageError, WriteConflictError, StorageCorruptionError, StorageFormatVersionError, OpfsUncertainOutcomeError, } from "../storage/types.js";
+import { MAX_SNAPSHOT_STREAM_CHUNK_BYTES } from "../storage/snapshot.js";
+import { parseRpcResponse, MAX_DATABASE_RPC_IN_FLIGHT, protocolVersion, } from "../worker-protocol/index.js";
 import { toColumnarBatch } from "./batch.js";
-import { CompactionJobCancelledError, CompactionMemoryBudgetError, CompactionWriteAmplificationError, MissingKeyError, SqlCompileError, UniqueConstraintError, } from "./errors.js";
+import { CompactionJobCancelledError, CompactionMemoryBudgetError, CompactionWriteAmplificationError, MaintenanceBacklogError, MissingKeyError, SqlCompileError, UniqueConstraintError, VisibleSegmentCursorStaleError, } from "./errors.js";
 import { QueryMemoryBudgetError } from "./memory.js";
 import { decodeQueryResult } from "./result-wire.js";
 import { serializeSchema } from "./schema-wire.js";
@@ -10,24 +11,56 @@ import { serializeSchema } from "./schema-wire.js";
  * database is tens of round trips, small enough that no single structured clone is long enough
  * to be felt as a dropped frame.
  */
-const SNAPSHOT_CHUNK_BYTES = 4 * 1024 * 1024;
+const SNAPSHOT_CHUNK_BYTES = MAX_SNAPSHOT_STREAM_CHUNK_BYTES;
+const MAX_SNAPSHOT_BYTE_ARRAY_LENGTH = 0xffffffff;
+async function* clientSnapshotChunks(bytes) {
+    for (let offset = 0; offset < bytes.byteLength; offset += SNAPSHOT_CHUNK_BYTES) {
+        yield bytes.subarray(offset, offset + SNAPSHOT_CHUNK_BYTES);
+    }
+}
+function throwIfClientSnapshotAborted(signal) {
+    if (signal?.aborted !== true)
+        return;
+    if (signal.reason instanceof Error)
+        throw signal.reason;
+    const error = new Error("Snapshot operation was aborted", { cause: signal.reason });
+    error.name = "AbortError";
+    throw error;
+}
 const errorRegistry = new Map([
     UniqueConstraintError,
     MissingKeyError,
+    CompactionBacklogError,
     CompactionMemoryBudgetError,
     CompactionWriteAmplificationError,
     CompactionJobCancelledError,
+    MaintenanceBacklogError,
     SqlCompileError,
     QueryMemoryBudgetError,
+    VisibleSegmentCursorStaleError,
+    BlockReadBatchTooLargeError,
     WriteConflictError,
+    SchemaConflictError,
     UniqueKeyConflictError,
+    UniqueIndexCoverageError,
     TableRecordConflictError,
     CompactionJobConflictError,
     GarbageCollectionJobConflictError,
+    IndexedDbSchemaUpgradeBlockedError,
     SnapshotManifestMissingError,
+    SnapshotImportConflictError,
+    TableInUseError,
     TransactionRecordConflictError,
     LeaseConflictError,
+    LeaseExpiredError,
+    LeaseOwnerConflictError,
+    StorageResourceLimitError,
     TempOwnerConflictError,
+    UniqueKeyBuildConflictError,
+    PostingBuildConflictError,
+    StorageCorruptionError,
+    StorageFormatVersionError,
+    OpfsUncertainOutcomeError,
 ].map((constructor) => [constructor.name, constructor]));
 function rehydrateError(serialized) {
     const constructor = errorRegistry.get(serialized.name);
@@ -63,17 +96,20 @@ export class MinnowDatabaseClient {
     #fatal;
     #closed = false;
     #onVisibilityChange;
+    #onMessage = (event) => {
+        this.#receive(event.data);
+    };
+    #onError = () => {
+        this.#fail(new Error("The database worker failed; see the worker's own error output"));
+    };
+    #onMessageError = () => {
+        this.#fail(new Error("A database worker message could not be deserialized"));
+    };
     constructor(transport, options = {}) {
         this.#transport = transport;
-        transport.addEventListener("message", (event) => {
-            this.#receive(event.data);
-        });
-        transport.addEventListener("error", () => {
-            this.#fail(new Error("The database worker failed; see the worker's own error output"));
-        });
-        transport.addEventListener("messageerror", () => {
-            this.#fail(new Error("A database worker message could not be deserialized"));
-        });
+        transport.addEventListener("message", this.#onMessage);
+        transport.addEventListener("error", this.#onError);
+        transport.addEventListener("messageerror", this.#onMessageError);
         const payload = {
             store: options.store ?? { kind: "indexeddb", name: "minnow" },
             ...(options.databaseOptions === undefined ? {} : { options: options.databaseOptions }),
@@ -143,6 +179,9 @@ export class MinnowDatabaseClient {
     async deleteBatch(tableName, input) {
         return (await this.#call("deleteBatch", [tableName, input]));
     }
+    async delete(tableName, key) {
+        return (await this.#call("delete", [tableName, key]));
+    }
     bufferedWriter(tableName, options = {}) {
         const handleId = crypto.randomUUID();
         const { onError, ...wireOptions } = options;
@@ -163,6 +202,50 @@ export class MinnowDatabaseClient {
     async query(sql, options) {
         return decodeQueryResult(await this.#call("query", options === undefined ? [sql] : [sql, options]));
     }
+    /** Pulls one transferred columnar page at a time, preserving worker backpressure. */
+    queryCursor(sql, options = {}) {
+        const call = this.#call.bind(this);
+        const invoke = this._invoke.bind(this);
+        const handleId = crypto.randomUUID();
+        const { signal, onStats, ...wireOptions } = options;
+        if (onStats !== undefined) {
+            throw new TypeError("Query cursor onStats callbacks are not available across a worker");
+        }
+        async function* batches() {
+            let opened = false;
+            let closing;
+            const close = () => {
+                if (!opened)
+                    return Promise.resolve();
+                closing ??= invoke(handleId, "close", []).catch(() => undefined);
+                return closing;
+            };
+            const onAbort = () => {
+                void close();
+            };
+            signal?.addEventListener("abort", onAbort, { once: true });
+            try {
+                signal?.throwIfAborted();
+                await call("queryCursorOpen", [handleId, sql, wireOptions]);
+                opened = true;
+                if (signal?.aborted === true) {
+                    await close();
+                    signal.throwIfAborted();
+                }
+                for (;;) {
+                    const next = (await invoke(handleId, "next", []));
+                    if (next.done)
+                        return undefined;
+                    yield decodeQueryResult(next.batch);
+                }
+            }
+            finally {
+                signal?.removeEventListener("abort", onAbort);
+                await close();
+            }
+        }
+        return batches();
+    }
     async run(query) {
         return decodeQueryResult(await this.#call("run", [query])).rows;
     }
@@ -231,30 +314,56 @@ export class MinnowDatabaseClient {
      * database; `onProgress` reports each slice as it lands.
      */
     async exportSnapshot(options = {}) {
+        const chunks = [];
+        let byteLength = 0;
+        for await (const chunk of this.exportSnapshotStream(options)) {
+            byteLength += chunk.byteLength;
+            if (!Number.isSafeInteger(byteLength) || byteLength > MAX_SNAPSHOT_BYTE_ARRAY_LENGTH) {
+                throw new RangeError("Snapshot exceeds the byte-array API's 4 GiB limit");
+            }
+            chunks.push(chunk);
+        }
+        const bytes = new Uint8Array(byteLength);
+        let offset = 0;
+        for (const chunk of chunks) {
+            bytes.set(chunk, offset);
+            offset += chunk.byteLength;
+        }
+        return bytes;
+    }
+    /** Pulls at most one bounded worker chunk at a time; closing releases the worker-side pin. */
+    async *exportSnapshotStream(options = {}) {
+        throwIfClientSnapshotAborted(options.signal);
         options.onProgress?.({ phase: "reading", transferredBytes: 0, totalBytes: 0 });
         const opened = (await this.#call("exportSnapshotOpen", []));
-        const totalBytes = opened.byteLength;
-        const bytes = new Uint8Array(totalBytes);
+        let transferredBytes = 0;
+        let succeeded = false;
         try {
-            let offset = 0;
-            while (offset < totalBytes) {
-                const chunk = (await this._invoke(opened.handleId, "read", [
-                    offset,
-                    Math.min(SNAPSHOT_CHUNK_BYTES, totalBytes - offset),
-                ]));
-                // A slice that came back empty would loop forever; the export is over either way.
-                if (chunk.byteLength === 0)
-                    throw new Error("Snapshot transfer returned no bytes");
-                bytes.set(chunk, offset);
-                offset += chunk.byteLength;
-                options.onProgress?.({ phase: "transfer", transferredBytes: offset, totalBytes });
+            for (;;) {
+                throwIfClientSnapshotAborted(options.signal);
+                const next = (await this._invoke(opened.handleId, "read", []));
+                if (next.done)
+                    break;
+                throwIfClientSnapshotAborted(options.signal);
+                if (!(next.chunk instanceof Uint8Array) || next.chunk.byteLength === 0) {
+                    throw new Error("Snapshot transfer returned an invalid chunk");
+                }
+                yield next.chunk;
+                transferredBytes += next.chunk.byteLength;
+                options.onProgress?.({ phase: "transfer", transferredBytes, totalBytes: 0 });
             }
+            succeeded = true;
         }
         finally {
             await this._invoke(opened.handleId, "close", []).catch(() => undefined);
+            if (succeeded) {
+                options.onProgress?.({
+                    phase: "done",
+                    transferredBytes,
+                    totalBytes: transferredBytes,
+                });
+            }
         }
-        options.onProgress?.({ phase: "done", transferredBytes: totalBytes, totalBytes });
-        return bytes;
     }
     /**
      * Loads a snapshot file into the worker's store, which must be empty. The bytes go over in the
@@ -262,6 +371,10 @@ export class MinnowDatabaseClient {
      * multi-megabyte restore can be shown moving rather than as a frozen tab.
      */
     async importSnapshot(bytes, options = {}) {
+        await this.importSnapshotStream(clientSnapshotChunks(bytes), options);
+    }
+    /** Uploads with one acknowledged worker chunk in flight; cancellation leaves resumable state. */
+    async importSnapshotStream(source, options = {}) {
         const handleId = crypto.randomUUID();
         const { onProgress } = options;
         this.#events.set(handleId, {
@@ -274,17 +387,25 @@ export class MinnowDatabaseClient {
                 }),
         });
         try {
+            throwIfClientSnapshotAborted(options.signal);
             await this.#call("importSnapshotOpen", [handleId]);
-            for (let offset = 0; offset < bytes.byteLength; offset += SNAPSHOT_CHUNK_BYTES) {
-                // Copied, not viewed: structured clone copies the buffer behind a typed array, so a
-                // subarray would post the whole file on every write.
-                await this._invoke(handleId, "write", [bytes.slice(offset, offset + SNAPSHOT_CHUNK_BYTES)]);
+            for await (const input of source) {
+                throwIfClientSnapshotAborted(options.signal);
+                if (!(input instanceof Uint8Array)) {
+                    throw new TypeError("Snapshot stream chunks must be Uint8Array values");
+                }
+                for (let offset = 0; offset < input.byteLength; offset += SNAPSHOT_CHUNK_BYTES) {
+                    // Copy the bounded window: cloning a subarray would otherwise transfer its whole backing
+                    // file on every request.
+                    const chunk = input.slice(offset, offset + SNAPSHOT_CHUNK_BYTES);
+                    await this.#post("rpc-call", handleId, "write", [chunk], [chunk.buffer]);
+                    throwIfClientSnapshotAborted(options.signal);
+                }
             }
             await this._invoke(handleId, "finish", []);
         }
         catch (error) {
-            // The handle survives a failed write; drop it rather than leaving the chunks in the worker.
-            await this._invoke(handleId, "close", []).catch(() => undefined);
+            await this._invoke(handleId, options.signal?.aborted === true ? "cancel" : "close", []).catch(() => undefined);
             throw error;
         }
         finally {
@@ -295,14 +416,31 @@ export class MinnowDatabaseClient {
     async bufferPoolStats() {
         return (await this.#call("bufferPoolStats", []));
     }
+    async maintenanceStatus() {
+        return (await this.#call("maintenanceStatus", []));
+    }
+    async checkIntegrity(options = {}) {
+        return (await this.#call("checkIntegrity", [options]));
+    }
+    async storageStats() {
+        return (await this.#call("storageStats", []));
+    }
+    async inspectInterruptedImport() {
+        return (await this.#call("inspectInterruptedImport", []));
+    }
+    async abortInterruptedImport(identity) {
+        return (await this.#call("abortInterruptedImport", [
+            identity,
+        ]));
+    }
     liveQueries(options = {}) {
         const handleId = crypto.randomUUID();
         const created = this.#call("liveQueries", [handleId, options]);
         return new ClientLiveQuerySet(this, handleId, created);
     }
     // --- Maintenance ------------------------------------------------------------------------------
-    async listVisibleSegments(tableName, version) {
-        return (await this.#call("listVisibleSegments", version === undefined ? [tableName] : [tableName, version]));
+    async listVisibleSegmentPage(tableName, options) {
+        return (await this.#call("listVisibleSegmentPage", options === undefined ? [tableName] : [tableName, options]));
     }
     async cleanupQuerySpill(options) {
         return (await this.#call("cleanupQuerySpill", options === undefined ? [] : [options]));
@@ -348,6 +486,9 @@ export class MinnowDatabaseClient {
         }
         finally {
             this.#events.clear();
+            this.#transport.removeEventListener?.("message", this.#onMessage);
+            this.#transport.removeEventListener?.("error", this.#onError);
+            this.#transport.removeEventListener?.("messageerror", this.#onMessageError);
             if (options.terminateWorker === true)
                 this.#transport.terminate?.();
         }
@@ -355,6 +496,8 @@ export class MinnowDatabaseClient {
     // --- Channel internals ------------------------------------------------------------------------
     /** @internal */
     async _invoke(handleId, method, args) {
+        if (this.#closed)
+            throw new Error("Database client is closed");
         return this.#post("rpc-call", handleId, method, args);
     }
     /** @internal */
@@ -370,15 +513,18 @@ export class MinnowDatabaseClient {
             throw new Error("Database client is closed");
         return this.#post("rpc-call", null, method, args);
     }
-    async #post(kind, handleId, method, args) {
+    async #post(kind, handleId, method, args, transfer, bypassLimit = kind === "rpc-init" || method === "dispose") {
         if (this.#fatal !== undefined)
             throw this.#fatal;
+        if (!bypassLimit && this.#pending.size >= MAX_DATABASE_RPC_IN_FLIGHT) {
+            throw new RangeError(`A database worker connection cannot hold more than ${String(MAX_DATABASE_RPC_IN_FLIGHT)} in-flight requests`);
+        }
         const requestId = crypto.randomUUID();
         return new Promise((resolve, reject) => {
             this.#pending.set(requestId, { resolve, reject });
             this.#transport.postMessage(kind === "rpc-init"
                 ? { version: protocolVersion, requestId, kind, payload: args[0] }
-                : { version: protocolVersion, requestId, kind, handleId, method, args });
+                : { version: protocolVersion, requestId, kind, handleId, method, args }, transfer === undefined ? undefined : { transfer });
         });
     }
     #receive(message) {
@@ -391,6 +537,9 @@ export class MinnowDatabaseClient {
                 return;
             if (response.event === "change")
                 route.onChange?.(decodeQueryResult(response.payload));
+            else if (response.event === "invalidate") {
+                route.onInvalidate?.(response.payload);
+            }
             else if (response.event === "error") {
                 route.onError?.(rehydrateError(response.payload));
             }
@@ -415,6 +564,7 @@ export class MinnowDatabaseClient {
         this.#fatal = error;
         const pending = [...this.#pending.values()];
         this.#pending.clear();
+        this.#events.clear();
         for (const call of pending)
             call.reject(error);
     }
@@ -507,6 +657,33 @@ export class ClientLiveQuerySet {
             throw error;
         }
     }
+    /** Registers dependency observation while leaving execution/result mapping to an adapter. */
+    async observe(query, options) {
+        await this.#created;
+        const subscriptionId = crypto.randomUUID();
+        const state = { completed: false };
+        this.client._routeEvents(subscriptionId, {
+            onInvalidate: options.onInvalidate.bind(options),
+            ...(options.onError === undefined ? {} : { onError: options.onError.bind(options) }),
+            onComplete: () => {
+                state.completed = true;
+                this.#subscriptionIds.delete(subscriptionId);
+                options.onComplete?.();
+            },
+        });
+        try {
+            const created = (await this.client._invoke(this.handleId, "observe", [
+                subscriptionId,
+                query,
+            ]));
+            this.#subscriptionIds.add(subscriptionId);
+            return new ClientLiveSubscription(this.client, subscriptionId, created.dependencyTableIds, () => this.#subscriptionIds.delete(subscriptionId), state);
+        }
+        catch (error) {
+            this.client._unrouteEvents(subscriptionId);
+            throw error;
+        }
+    }
     async stats() {
         await this.#created;
         return (await this.client._invoke(this.handleId, "stats", []));
@@ -560,4 +737,3 @@ export class ClientLiveSubscription {
         }
     }
 }
-//# sourceMappingURL=client.js.map