hyperdb-wasm 0.0.26792-dev.wasm.g72cc44c1 → 0.0.26862-dev.wasm.g42f1012b

package/dist/node_worker.js

@@ -1,14 +1,21 @@
 // src/node_worker.ts
-import { createReadStream } from "node:fs";
+import { closeSync, createReadStream, fstatSync, openSync, read } from "node:fs";
+import { opendir as opendir2, stat } from "node:fs/promises";
 import { createRequire } from "node:module";
-import { dirname, isAbsolute, join, posix, relative, resolve, sep } from "node:path";
+import { basename as basename2, dirname, isAbsolute as isAbsolute2, join as join2, posix, resolve as resolve2 } from "node:path";
+import { randomUUID as randomUUID2 } from "node:crypto";
 import { Readable } from "node:stream";
 import { parentPort, workerData } from "node:worker_threads";
 import { fileURLToPath } from "node:url";
 import { createBrotliDecompress } from "node:zlib";
 
 // src/wasm_bindings.ts
-var resultStates = ["ok", "pending", "chunk", "done", "error", "busy"];
+function completionFor(cleanups) {
+  const pending = cleanups.filter((cleanup) => cleanup !== void 0);
+  return pending.length === 0 ? void 0 : Promise.all(pending).then(() => void 0);
+}
+var resultStates = ["ok", "pending", "chunk", "done", "error", "busy", "structuredError"];
+var structuredErrorMagic = new Uint8Array([72, 68, 66, 69]);
 function safeInteger(value, name) {
   const number = typeof value === "bigint" && value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value;
   if (typeof number !== "number" || !Number.isSafeInteger(number) || number < 0) throw new Error(`${name} must be a nonnegative safe integer`);
@@ -18,6 +25,33 @@ function validateCString(value, name) {
   if (!value) throw new Error(`${name} must not be empty`);
   if (value.includes("\0")) throw new Error(`${name} must not contain NUL`);
 }
+function decodeStructuredError(payload) {
+  if (!structuredErrorMagic.every((byte, index) => payload[index] === byte)) throw new Error("structured error has invalid magic");
+  const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
+  if (payload.byteLength < 8 || view.getUint32(4, true) !== 1) throw new Error("unsupported structured error version");
+  let offset = 8;
+  const decoder = new TextDecoder("utf-8", { fatal: true });
+  const readField = () => {
+    if (offset + 4 > payload.byteLength) throw new Error("truncated structured error field");
+    const length = view.getUint32(offset, true);
+    offset += 4;
+    if (length > payload.byteLength - offset) throw new Error("truncated structured error field");
+    const value = decoder.decode(payload.subarray(offset, offset + length));
+    offset += length;
+    return value;
+  };
+  const sqlState = readField();
+  const errorSource = readField();
+  const message = readField();
+  const detail = readField();
+  const hint = readField();
+  if (offset !== payload.byteLength) throw new Error("structured error has trailing bytes");
+  if (!/^[0-9A-Z]{5}$/.test(sqlState)) throw new Error("structured error has invalid SQLSTATE");
+  if (errorSource !== "User" && errorSource !== "System" && errorSource !== "OverloadRejection") {
+    throw new Error("structured error has invalid source");
+  }
+  return { sqlState, errorSource, message, detail, hint };
+}
 var HyperDBBindings = class {
   module;
   initialize;
@@ -29,13 +63,18 @@ var HyperDBBindings = class {
   checkpointDatabase;
   dropDatabase;
   shutdown;
+  registerExternalFile;
+  registerExternalDirectory;
+  removeExternalRegistration;
   attachDatabase;
   detachDatabase;
   startQuery;
+  startQueryWithInfo;
   insertArrowIPCFromPath;
   pollQuery;
   cancelQuery;
   releaseQuery;
+  releaseQueryWithInfo;
   resultState;
   resultPayloadLength;
   free;
@@ -51,13 +90,18 @@ var HyperDBBindings = class {
     this.checkpointDatabase = module2.cwrap("hyperdb_wasm_checkpoint_database", "pointer", ["string"]);
     this.dropDatabase = module2.cwrap("hyperdb_wasm_drop_database", "pointer", ["string"]);
     this.shutdown = module2.cwrap("hyperdb_wasm_shutdown", "pointer", []);
+    this.registerExternalFile = module2.cwrap("hyperdb_wasm_register_external_file", "pointer", ["string", "string", "number"]);
+    this.registerExternalDirectory = module2.cwrap("hyperdb_wasm_register_external_directory", "pointer", ["string", "string", "number"]);
+    this.removeExternalRegistration = module2.cwrap("hyperdb_wasm_remove_external_registration", "pointer", ["string"]);
     this.attachDatabase = module2.cwrap("hyperdb_wasm_attach_database", "pointer", ["number", "string", "string"]);
     this.detachDatabase = module2.cwrap("hyperdb_wasm_detach_database", "pointer", ["number", "string"]);
-    this.startQuery = module2.cwrap("hyperdb_wasm_query_start", "pointer", ["number", "string"]);
+    this.startQuery = module2.cwrap("hyperdb_wasm_query_start", "pointer", ["number", "string", "number"]);
+    this.startQueryWithInfo = module2.cwrap("hyperdb_wasm_query_start_with_info", "pointer", ["number", "string"]);
     this.insertArrowIPCFromPath = module2.cwrap("hyperdb_wasm_arrow_insert_from_path", "pointer", ["number", "string", "string", "string", "number"]);
     this.pollQuery = module2.cwrap("hyperdb_wasm_query_poll", "pointer", ["number"]);
     this.cancelQuery = module2.cwrap("hyperdb_wasm_query_cancel", "pointer", ["number"]);
     this.releaseQuery = module2.cwrap("hyperdb_wasm_query_release", "pointer", ["number"]);
+    this.releaseQueryWithInfo = module2.cwrap("hyperdb_wasm_query_release_with_info", "pointer", ["number"]);
     this.resultState = module2.cwrap("hyperdb_wasm_result_state", "number", ["number"]);
     this.resultPayloadLength = module2.cwrap("hyperdb_wasm_result_payload_len", "number", ["number"]);
     this.free = module2.cwrap("hyperdb_wasm_free", null, ["pointer"]);
@@ -73,8 +117,92 @@ var HyperDBBindings = class {
       this.module.FS.unlink(request.path);
       return { response: { id: request.id, type: "response", result: { state: "ok", payload: new Uint8Array() } }, transfer: [] };
     }
+    if (request.type === "registerExternalFile" && "path" in request) {
+      if (!this.module.externalFileBackend) {
+        return { response: { id: request.id, type: "response", result: { state: "error", error: "Host external files are supported only by the Node client" } }, transfer: [] };
+      }
+      const registration = this.module.externalFileBackend.register(request.path);
+      try {
+        const pointer2 = safeInteger(this.registerExternalFile(registration.name, registration.locator, registration.size), "result pointer");
+        const result2 = this.decodeResult(pointer2);
+        if (result2.state === "ok") {
+          const path = new TextDecoder("utf-8", { fatal: true }).decode(result2.payload);
+          try {
+            this.module.externalFileBackend.registered(path, registration.locator);
+          } catch (error) {
+            const cleanupPointer = safeInteger(this.removeExternalRegistration(path), "result pointer");
+            this.decodeResult(cleanupPointer);
+            throw error;
+          }
+        } else {
+          this.module.externalFileBackend.removed(registration.locator);
+        }
+        return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
+      } catch (error) {
+        this.module.externalFileBackend.removed(registration.locator);
+        throw error;
+      }
+    }
+    if (request.type === "registerExternalFile" && !this.module.externalFileBackendAvailable) {
+      return { response: { id: request.id, type: "response", result: { state: "error", error: "Blob external files are supported only by the browser client" } }, transfer: [] };
+    }
+    if (request.type === "registerExternalDirectory") {
+      const backend = this.module.externalDirectoryBackend;
+      if (!backend) {
+        return { response: { id: request.id, type: "response", result: { state: "error", error: "External directories are not supported by this client" } }, transfer: [] };
+      }
+      const registration = backend.register("handle" in request ? request.handle : request.path, request.mode);
+      try {
+        const pointer2 = safeInteger(this.registerExternalDirectory(registration.name, registration.locator, Number(request.mode === "readwrite")), "result pointer");
+        const result2 = this.decodeResult(pointer2);
+        if (result2.state === "ok") {
+          const path = new TextDecoder("utf-8", { fatal: true }).decode(result2.payload);
+          try {
+            backend.registered(path, registration.locator);
+          } catch (error) {
+            const cleanupPointer = safeInteger(this.removeExternalRegistration(path), "result pointer");
+            this.decodeResult(cleanupPointer);
+            throw error;
+          }
+        } else {
+          const completion = completionFor([backend.removed(registration.locator)]);
+          return { response: { id: request.id, type: "response", result: result2 }, transfer: [], ...completion && { completion } };
+        }
+        return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
+      } catch (error) {
+        void Promise.resolve(backend.removed(registration.locator)).catch(() => void 0);
+        throw error;
+      }
+    }
+    if (request.type === "removeExternalRegistration" && (this.module.externalFileBackend || this.module.externalDirectoryBackend)) {
+      validateCString(request.path, "external registration path");
+      const pointer2 = safeInteger(this.removeExternalRegistration(request.path), "result pointer");
+      const result2 = this.decodeResult(pointer2);
+      if (result2.state === "ok") {
+        const completion = completionFor([
+          this.module.externalFileBackend?.removed(request.path),
+          this.module.externalDirectoryBackend?.removed(request.path)
+        ]);
+        return { response: { id: request.id, type: "response", result: result2 }, transfer: [], ...completion && { completion } };
+      }
+      return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
+    }
+    if (request.type === "removeExternalRegistration" && !this.module.externalFileBackendAvailable) {
+      return { response: { id: request.id, type: "response", result: { state: "error", error: "External files are not supported by this client" } }, transfer: [] };
+    }
     const pointer = safeInteger(this.invoke(request), "result pointer");
     const result = this.decodeResult(pointer);
+    if (request.type === "shutdown" && result.state === "ok") {
+      const completion = completionFor([
+        this.module.externalFileBackend?.shutdown(),
+        this.module.externalDirectoryBackend?.shutdown()
+      ]);
+      return {
+        response: { id: request.id, type: "response", result },
+        transfer: [],
+        ...completion && { completion }
+      };
+    }
     const response = { id: request.id, type: "response", result };
     return { response, transfer: result.state === "chunk" ? [result.payload.buffer] : [] };
   }
@@ -110,8 +238,20 @@ var HyperDBBindings = class {
         return this.detachDatabase(request.connection, request.alias);
       case "shutdown":
         return this.shutdown();
+      case "registerExternalFile":
+        if ("path" in request) throw new Error("host external file request has no backend");
+        validateCString(request.name, "external file name");
+        validateCString(request.url, "external file URL");
+        return this.registerExternalFile(request.name, request.url, safeInteger(request.size, "external file size"));
+      case "registerExternalDirectory":
+        throw new Error("external directory request has no backend");
+      case "removeExternalRegistration":
+        validateCString(request.path, "external registration path");
+        return this.removeExternalRegistration(request.path);
       case "startQuery":
-        return this.startQuery(request.connection, request.sql);
+        return this.startQuery(request.connection, request.sql, Number(request.sqlTestMode ?? false));
+      case "startQueryWithInfo":
+        return this.startQueryWithInfo(request.connection, request.sql);
       case "insertArrowIPCFromPath": {
         const path = request.internal ? request.path : this.module.resolveHyperDBPath?.(request.path) ?? request.path;
         return this.insertArrowIPCFromPath(request.connection, path, request.name, request.schema ?? "", Number(request.create));
@@ -122,6 +262,8 @@ var HyperDBBindings = class {
         return this.cancelQuery(request.query);
       case "releaseQuery":
         return this.releaseQuery(request.query);
+      case "releaseQueryWithInfo":
+        return this.releaseQueryWithInfo(request.query);
     }
   }
   decodeResult(pointer) {
@@ -151,6 +293,8 @@ var HyperDBBindings = class {
           return { state, error: new TextDecoder().decode(payload) };
         case "busy":
           return { state, error: new TextDecoder().decode(payload) };
+        case "structuredError":
+          return { state: "error", error: decodeStructuredError(payload) };
       }
     } finally {
       this.free(pointer);
@@ -190,6 +334,8 @@ function isEngineRequest(value) {
     case "detachDatabase":
       return hasKeys(value, ["id", "type", "connection", "alias"]) && Number.isSafeInteger(value.connection) && typeof value.alias === "string";
     case "startQuery":
+      return (hasKeys(value, ["id", "type", "connection", "sql"]) || hasKeys(value, ["id", "type", "connection", "sql", "sqlTestMode"])) && Number.isSafeInteger(value.connection) && typeof value.sql === "string" && (value.sqlTestMode === void 0 || typeof value.sqlTestMode === "boolean");
+    case "startQueryWithInfo":
       return hasKeys(value, ["id", "type", "connection", "sql"]) && Number.isSafeInteger(value.connection) && typeof value.sql === "string";
     case "insertArrowIPCFromPath":
       return hasKeys(value, ["id", "type", "connection", "path", "name", "schema", "create", "internal"]) && Number.isSafeInteger(value.connection) && typeof value.path === "string" && typeof value.name === "string" && (typeof value.schema === "string" || value.schema === null) && typeof value.create === "boolean" && typeof value.internal === "boolean";
@@ -197,9 +343,16 @@ function isEngineRequest(value) {
       return hasKeys(value, ["id", "type", "bytes"]) && value.bytes instanceof Uint8Array;
     case "removeFile":
       return hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
+    case "registerExternalFile":
+      return hasKeys(value, ["id", "type", "name", "url", "size"]) && typeof value.name === "string" && typeof value.url === "string" && typeof value.size === "number" && Number.isSafeInteger(value.size) && value.size >= 0 || hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
+    case "registerExternalDirectory":
+      return (value.mode === "read" || value.mode === "readwrite") && (hasKeys(value, ["id", "type", "handle", "mode"]) && isRecord(value.handle) && value.handle.kind === "directory" || hasKeys(value, ["id", "type", "path", "mode"]) && typeof value.path === "string" && !!value.path && !value.path.includes("\0"));
+    case "removeExternalRegistration":
+      return hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
     case "pollQuery":
     case "cancelQuery":
     case "releaseQuery":
+    case "releaseQueryWithInfo":
       return hasKeys(value, ["id", "type", "query"]) && Number.isSafeInteger(value.query);
     default:
       return false;
@@ -210,6 +363,10 @@ function isEngineRequest(value) {
 function errorText(error) {
   return error instanceof Error ? error.message : String(error);
 }
+function isNotFoundError(error) {
+  if (typeof error !== "object" || error === null) return false;
+  return "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR") || "name" in error && error.name === "NotFoundError";
+}
 function safePointer(value, name) {
   const pointer = typeof value === "bigint" && value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value;
   if (typeof pointer !== "number" || !Number.isSafeInteger(pointer) || pointer < 0) throw new Error(`${name} must be a nonnegative safe integer`);
@@ -218,22 +375,48 @@ function safePointer(value, name) {
 function allocate(module2, bytes) {
   const pointer = safePointer(module2._malloc(bytes.byteLength || 1), "allocation pointer");
   if (pointer === 0) throw new Error("wasm allocation failed");
-  if (pointer + bytes.byteLength > module2.HEAPU8.byteLength) throw new Error("allocation exceeds the wasm heap");
-  module2.HEAPU8.set(bytes, pointer);
+  module2.getValue(pointer, "i8");
+  const heap = module2.HEAPU8;
+  if (pointer + bytes.byteLength > heap.byteLength) throw new Error("allocation exceeds the wasm heap");
+  heap.set(bytes, pointer);
   return pointer;
 }
+function externalDirectoryError(module2, error) {
+  return safePointer(module2.stringToNewUTF8(errorText(error)), "external directory error pointer");
+}
+function completeExternalDirectoryOperation(module2, id, complete, operation) {
+  void Promise.resolve().then(operation).then(
+    () => complete(id, 0),
+    (error) => complete(id, externalDirectoryError(module2, error))
+  );
+}
 function installHttpRuntime(module2, transport2) {
-  module2.hyperdbWasmFetchStart = (id, url, offset, length) => {
+  module2.hyperdbWasmFetchStart = (id, url, offset, length, expectedSize) => {
+    if (url.startsWith("hyperdb-node-file:") || url.startsWith("hyperdb-node-host-file:") || url.startsWith("hyperdb-node-directory-file:") || url.startsWith("hyperdb-browser-file:")) {
+      const read2 = module2.externalFileBackend?.read(url, offset, length, expectedSize) ?? module2.externalDirectoryBackend?.read(url, offset, length, expectedSize);
+      if (!read2) {
+        module2._hyperdb_wasm_fetch_complete(id, 0, -1);
+        return;
+      }
+      void read2.then((bytes) => {
+        module2._hyperdb_wasm_fetch_complete(id, allocate(module2, bytes), bytes.byteLength);
+      }).catch(() => {
+        module2._hyperdb_wasm_fetch_complete(id, 0, -1);
+      });
+      return;
+    }
     const headers = new Headers();
     if (length > 0) headers.set("Range", `bytes=${offset}-${offset + length - 1}`);
     void fetch(url, { headers, redirect: "manual" }).then(async (response) => {
       if (!response.ok || response.type === "opaqueredirect" || (length === 0 ? response.status !== 200 : response.status !== 206)) {
+        void response.body?.cancel().catch(() => void 0);
         module2._hyperdb_wasm_fetch_complete(id, 0, -1);
         return;
       }
       if (length > 0) {
         const contentRange = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(response.headers.get("content-range") ?? "");
-        if (!contentRange || contentRange[1] !== String(offset) || contentRange[2] !== String(offset + length - 1)) {
+        if (!contentRange || contentRange[1] !== String(offset) || contentRange[2] !== String(offset + length - 1) || expectedSize >= 0 && contentRange[3] !== String(expectedSize)) {
+          void response.body?.cancel().catch(() => void 0);
           module2._hyperdb_wasm_fetch_complete(id, 0, -1);
           return;
         }
@@ -282,6 +465,97 @@ function installHttpRuntime(module2, transport2) {
     });
   };
 }
+function installExternalDirectoryRuntime(module2) {
+  const backend = module2.externalDirectoryBackend;
+  if (!backend) return;
+  const listComplete = module2._hyperdb_wasm_external_directory_list_complete;
+  const openComplete = module2._hyperdb_wasm_external_directory_open_complete;
+  if (listComplete) {
+    module2.hyperdbWasmExternalDirectoryListStart = (id, locator) => {
+      void backend.list(locator).then((paths) => {
+        listComplete(id, safePointer(module2.stringToNewUTF8(JSON.stringify(paths)), "external directory list pointer"), 0);
+      }).catch((error) => {
+        listComplete(id, 0, externalDirectoryError(module2, error));
+      });
+    };
+  }
+  if (openComplete) {
+    module2.hyperdbWasmExternalDirectoryOpenStart = (id, locator, path) => {
+      void backend.open(locator, path).then((file) => {
+        openComplete(id, safePointer(module2.stringToNewUTF8(file.locator), "external file locator pointer"), file.size);
+      }).catch((error) => openComplete(id, 0, isNotFoundError(error) ? -2 : -1));
+    };
+  }
+  const createDirectoriesComplete = module2._hyperdb_wasm_external_directory_create_directories_complete;
+  if (createDirectoriesComplete) {
+    module2.hyperdbWasmExternalDirectoryCreateDirectoriesStart = (id, locator, path) => {
+      completeExternalDirectoryOperation(module2, id, createDirectoriesComplete, () => backend.createDirectories(locator, path));
+    };
+  }
+  const writerWriteComplete = module2._hyperdb_wasm_external_directory_writer_write_complete;
+  if (writerWriteComplete) {
+    module2.hyperdbWasmExternalDirectoryWriterWriteStart = (writer, id, locator, path, offset, pointerValue, length) => {
+      try {
+        const pointer = safePointer(pointerValue, "external directory write pointer");
+        const safeLength = safePointer(length, "external directory write length");
+        const end = safePointer(pointer + safeLength, "external directory write end");
+        module2.getValue(pointer, "i8");
+        if (end > module2.HEAPU8.byteLength) throw new Error("external directory write exceeds the wasm heap");
+        const bytes = module2.HEAPU8.slice(pointer, end);
+        completeExternalDirectoryOperation(module2, id, writerWriteComplete, async () => {
+          await backend.openWriter(writer, locator, path);
+          await backend.write(writer, offset, bytes);
+        });
+      } catch (error) {
+        writerWriteComplete(id, externalDirectoryError(module2, error));
+      }
+    };
+  }
+  const writerFinishComplete = module2._hyperdb_wasm_external_directory_writer_finish_complete;
+  if (writerFinishComplete) {
+    module2.hyperdbWasmExternalDirectoryWriterFinishStart = (writer, id, locator, path, size) => {
+      completeExternalDirectoryOperation(module2, id, writerFinishComplete, async () => {
+        await backend.openWriter(writer, locator, path);
+        await backend.finish(writer, size);
+      });
+    };
+  }
+  const writerAbortComplete = module2._hyperdb_wasm_external_directory_writer_abort_complete;
+  if (writerAbortComplete) {
+    module2.hyperdbWasmExternalDirectoryWriterAbortStart = (writer, id) => {
+      completeExternalDirectoryOperation(module2, id, writerAbortComplete, () => backend.abort(writer));
+    };
+  }
+}
+function installHostFileRuntime(module2) {
+  const complete = module2._hyperdb_wasm_host_file_open_complete;
+  if (!complete) return;
+  module2.hyperdbWasmHostFileOpenStart = (id, path) => {
+    try {
+      const file = module2.externalFileBackend?.openHostFile(path);
+      if (!file) throw new Error("Node host-file backend is unavailable");
+      complete(id, file.id, file.size);
+    } catch (error) {
+      complete(id, 0, isNotFoundError(error) ? -2 : -1);
+    }
+  };
+  module2.hyperdbWasmHostFileClose = (id) => module2.externalFileBackend?.closeHostFile(id);
+  const listComplete = module2._hyperdb_wasm_external_directory_list_complete;
+  if (listComplete) {
+    module2.hyperdbWasmHostDirectoryListStart = (id, path) => {
+      const list = module2.externalFileBackend?.listHostDirectory(path);
+      if (!list) {
+        listComplete(id, 0, externalDirectoryError(module2, new Error("Node host-directory backend is unavailable")));
+        return;
+      }
+      void list.then((paths) => {
+        listComplete(id, safePointer(module2.stringToNewUTF8(JSON.stringify(paths)), "host directory list pointer"), 0);
+      }).catch((error) => {
+        listComplete(id, 0, externalDirectoryError(module2, error));
+      });
+    };
+  }
+}
 function configureWorker(module2, transport2, configureFilesystem = () => void 0) {
   const queuedRequests = [];
   let bindings;
@@ -292,8 +566,14 @@ function configureWorker(module2, transport2, configureFilesystem = () => void 0
     }
     try {
       const result = bindings.handle(request);
-      if (request.type === "shutdown" && result.response.result.state === "ok") module2.PThread.terminateAllThreads();
-      transport2.postMessage(result.response, result.transfer);
+      const complete = () => {
+        if (request.type === "shutdown" && result.response.result.state === "ok") module2.PThread.terminateAllThreads();
+        transport2.postMessage(result.response, result.transfer);
+      };
+      if (result.completion) void result.completion.then(complete, (error) => {
+        transport2.postMessage({ id: request.id, type: "response", result: { state: "error", error: errorText(error) } });
+      });
+      else complete();
     } catch (error) {
       transport2.postMessage({ id: request.id, type: "response", result: { state: "error", error: errorText(error) } });
     }
@@ -307,6 +587,8 @@ function configureWorker(module2, transport2, configureFilesystem = () => void 0
     try {
       configureFilesystem(module2);
       installHttpRuntime(module2, transport2);
+      installHostFileRuntime(module2);
+      installExternalDirectoryRuntime(module2);
       bindings = new HyperDBBindings(module2);
       transport2.postMessage({ type: "ready" });
       for (const request of queuedRequests.splice(0)) handle(request);
@@ -317,17 +599,323 @@ function configureWorker(module2, transport2, configureFilesystem = () => void 0
   module2.onAbort = (reason) => transport2.postMessage({ type: "fatal", error: `engine aborted: ${String(reason)}` });
 }
 
+// src/node_external_directory.ts
+import { constants, lstatSync, realpathSync } from "node:fs";
+import { lstat, mkdir, open, opendir, rename, unlink } from "node:fs/promises";
+import { randomUUID } from "node:crypto";
+import { basename, isAbsolute, join, relative, resolve } from "node:path";
+var directoryLocatorPrefix = "hyperdb-node-directory:";
+var fileLocatorPrefix = "hyperdb-node-directory-file:";
+function validateSegment(segment) {
+  if (!segment || segment === "." || segment === ".." || segment.includes("/") || segment.includes("\\") || segment.includes("\0")) {
+    throw new Error("External directory entries must have safe relative path segments");
+  }
+}
+function relativeSegments(path) {
+  if (!path || isAbsolute(path) || path.endsWith("/") || path.includes("\\") || path.includes("\0")) {
+    throw new Error("External directory file path must be relative");
+  }
+  const segments = path.split("/");
+  for (const segment of segments) validateSegment(segment);
+  return segments;
+}
+function directorySegments(path) {
+  return path ? relativeSegments(path) : [];
+}
+function nonnegativeSafeInteger(value, name) {
+  if (!Number.isSafeInteger(value) || value < 0) throw new Error(`${name} must be a nonnegative safe integer`);
+}
+function isNotFound(error) {
+  return typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT";
+}
+function isAlreadyExists(error) {
+  return typeof error === "object" && error !== null && "code" in error && error.code === "EEXIST";
+}
+function confinedPath(root, segments) {
+  const path = join(root, ...segments);
+  const fromRoot = relative(root, path);
+  if (isAbsolute(fromRoot) || fromRoot === ".." || fromRoot.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)) {
+    throw new Error("External directory path escapes its registration");
+  }
+  return path;
+}
+async function ensureDirectory(root, segments, create) {
+  let current = root;
+  for (const segment of segments) {
+    current = join(current, segment);
+    const fromRoot = relative(root, current);
+    if (isAbsolute(fromRoot) || fromRoot === ".." || fromRoot.startsWith(`..${process.platform === "win32" ? "\\" : "/"}`)) {
+      throw new Error("External directory path escapes its registration");
+    }
+    try {
+      const info = await lstat(current);
+      if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("External directory path contains a non-directory entry");
+    } catch (error) {
+      if (!create || !isNotFound(error)) throw error;
+      await mkdir(current).catch((mkdirError) => {
+        if (!isAlreadyExists(mkdirError)) throw mkdirError;
+      });
+      const info = await lstat(current);
+      if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("External directory path contains a non-directory entry");
+    }
+  }
+  return current;
+}
+var NodeExternalDirectoryBackend = class {
+  directories = /* @__PURE__ */ new Map();
+  paths = /* @__PURE__ */ new Map();
+  files = /* @__PURE__ */ new Map();
+  writers = /* @__PURE__ */ new Map();
+  register(path, mode) {
+    if (typeof path !== "string" || !path || path.includes("\0")) throw new Error("External directory path is invalid");
+    const root = realpathSync(resolve(path));
+    const info = lstatSync(root);
+    if (info.isSymbolicLink() || !info.isDirectory()) throw new Error("External directory path must identify a directory");
+    const locator = `${directoryLocatorPrefix}${randomUUID()}`;
+    this.directories.set(locator, { root, mode, files: /* @__PURE__ */ new Map(), locators: /* @__PURE__ */ new Set(), temporaryPaths: /* @__PURE__ */ new Set() });
+    return { name: basename(root) || "root", locator };
+  }
+  registered(path, locator) {
+    if (!this.directories.has(locator)) throw new Error("External directory registration is unavailable");
+    this.paths.set(path, locator);
+  }
+  async removed(pathOrLocator) {
+    const locator = this.paths.get(pathOrLocator) ?? pathOrLocator;
+    const directory = this.directories.get(locator);
+    if (!directory) return;
+    const writerCleanup = [];
+    for (const [writer, state] of this.writers) {
+      if (state.locator !== locator) continue;
+      if (!state.terminal) writerCleanup.push(this.abort(writer));
+      else writerCleanup.push(state.chain.then(() => this.cleanupWriter(state)).finally(() => this.writers.delete(writer)));
+    }
+    await Promise.all(writerCleanup);
+    const fileCleanup = [];
+    for (const fileLocator of directory.locators) {
+      const file = this.files.get(fileLocator);
+      if (file) fileCleanup.push(file.descriptor.close());
+      this.files.delete(fileLocator);
+    }
+    await Promise.all(fileCleanup);
+    this.directories.delete(locator);
+    for (const [path, pathLocator] of this.paths) {
+      if (pathLocator === locator) this.paths.delete(path);
+    }
+  }
+  async list(locator) {
+    const directory = this.getDirectory(locator);
+    const files = [];
+    const visit = async (path, prefix) => {
+      const entries = await opendir(path);
+      for await (const entry of entries) {
+        validateSegment(entry.name);
+        const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+        const child = confinedPath(directory.root, relativeSegments(relativePath));
+        if (directory.temporaryPaths.has(child)) continue;
+        const info = await lstat(child);
+        if (info.isSymbolicLink()) throw new Error("External directory must not contain symbolic links");
+        if (info.isDirectory()) await visit(child, relativePath);
+        else if (info.isFile()) files.push(relativePath);
+        else throw new Error("External directory contains an unsupported entry");
+      }
+    };
+    await visit(directory.root, "");
+    files.sort();
+    return files;
+  }
+  async open(locator, path) {
+    const directory = this.getDirectory(locator);
+    const cached = directory.files.get(path);
+    if (cached) {
+      const file = await cached;
+      return { locator: file.locator, size: file.size };
+    }
+    const segments = relativeSegments(path);
+    const snapshot = (async () => {
+      await ensureDirectory(directory.root, segments.slice(0, -1), false);
+      const target = confinedPath(directory.root, segments);
+      const info = await lstat(target);
+      if (info.isSymbolicLink() || !info.isFile()) throw new Error("External directory entry is not a regular file");
+      const descriptor = await open(target, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0));
+      try {
+        const openedInfo = await descriptor.stat({ bigint: true });
+        if (!openedInfo.isFile()) throw new Error("External directory entry is not a regular file");
+        if (openedInfo.size > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("External file size must be a nonnegative safe integer");
+        const file = { locator: `${fileLocatorPrefix}${randomUUID()}`, descriptor, size: Number(openedInfo.size) };
+        this.files.set(file.locator, file);
+        directory.locators.add(file.locator);
+        return file;
+      } catch (error) {
+        await descriptor.close();
+        throw error;
+      }
+    })();
+    directory.files.set(path, snapshot);
+    try {
+      const file = await snapshot;
+      return { locator: file.locator, size: file.size };
+    } catch (error) {
+      directory.files.delete(path);
+      throw error;
+    }
+  }
+  read(locator, offset, length, expectedSize) {
+    if (!locator.startsWith(fileLocatorPrefix)) return void 0;
+    const file = this.files.get(locator);
+    if (!file) return Promise.reject(new Error("External directory file registration does not exist"));
+    const end = offset + length;
+    if (expectedSize !== file.size || !Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(length) || length < 0 || !Number.isSafeInteger(end) || end > file.size) {
+      return Promise.reject(new Error("External file byte range is invalid"));
+    }
+    return (async () => {
+      const result = new Uint8Array(length);
+      let bytesRead = 0;
+      while (bytesRead < length) {
+        const readResult = await file.descriptor.read(result, bytesRead, length - bytesRead, offset + bytesRead);
+        if (readResult.bytesRead === 0) throw new Error("External file ended before the registered size");
+        bytesRead += readResult.bytesRead;
+      }
+      return result;
+    })();
+  }
+  async createDirectories(locator, path) {
+    const directory = this.getWritableDirectory(locator);
+    await ensureDirectory(directory.root, directorySegments(path), true);
+  }
+  openWriter(writer, locator, path) {
+    nonnegativeSafeInteger(writer, "External directory writer");
+    const directory = this.getWritableDirectory(locator);
+    const segments = relativeSegments(path);
+    const target = confinedPath(directory.root, segments);
+    const existing = this.writers.get(writer);
+    if (existing) {
+      if (existing.locator !== locator || existing.path !== path) throw new Error("External directory writer was reused for another path");
+      return existing.chain;
+    }
+    const state = { locator, path, target, temp: void 0, descriptor: void 0, chain: Promise.resolve(), terminal: false };
+    this.writers.set(writer, state);
+    return this.enqueue(state, async () => {
+      const parent = await ensureDirectory(directory.root, segments.slice(0, -1), true);
+      try {
+        const info = await lstat(target);
+        if (info.isSymbolicLink() || !info.isFile()) throw new Error("External directory write target is not a regular file");
+      } catch (error) {
+        if (!isNotFound(error)) throw error;
+      }
+      state.temp = join(parent, `.hyperdb-${randomUUID()}.tmp`);
+      directory.temporaryPaths.add(state.temp);
+      try {
+        state.descriptor = await open(state.temp, constants.O_CREAT | constants.O_EXCL | constants.O_WRONLY, 384);
+      } catch (error) {
+        state.temp = void 0;
+        throw error;
+      }
+    });
+  }
+  write(writer, offset, bytes) {
+    nonnegativeSafeInteger(offset, "External directory write offset");
+    nonnegativeSafeInteger(offset + bytes.byteLength, "External directory write end");
+    const state = this.getOpenWriter(writer);
+    return this.enqueue(state, async () => {
+      if (!state.descriptor) throw new Error("External directory writer failed to open");
+      let written = 0;
+      while (written < bytes.byteLength) {
+        const result = await state.descriptor.write(bytes, written, bytes.byteLength - written, offset + written);
+        if (result.bytesWritten === 0) throw new Error("External directory write made no progress");
+        written += result.bytesWritten;
+      }
+    });
+  }
+  finish(writer, size) {
+    nonnegativeSafeInteger(size, "External directory file size");
+    const state = this.getOpenWriter(writer);
+    state.terminal = true;
+    const operation = this.enqueue(state, async () => {
+      let failure = state.failure;
+      if (failure === void 0 && (!state.descriptor || !state.temp)) failure = new Error("External directory writer failed to open");
+      if (failure === void 0) {
+        try {
+          await state.descriptor?.truncate(size);
+          await state.descriptor?.close();
+          state.descriptor = void 0;
+          await rename(state.temp, state.target);
+          state.temp = void 0;
+        } catch (error) {
+          failure = error;
+        }
+      }
+      if (failure !== void 0) {
+        await this.cleanupWriter(state);
+        throw failure;
+      }
+      this.getDirectory(state.locator).files.delete(state.path);
+    }, true);
+    return operation.finally(() => this.writers.delete(writer));
+  }
+  abort(writer) {
+    const state = this.getOpenWriter(writer);
+    state.terminal = true;
+    const operation = this.enqueue(state, () => this.cleanupWriter(state), true);
+    return operation.finally(() => this.writers.delete(writer));
+  }
+  async shutdown() {
+    await Promise.all([...this.directories.keys()].map((locator) => this.removed(locator)));
+    await Promise.all([...this.files.values()].map((file) => file.descriptor.close()));
+    this.files.clear();
+    this.paths.clear();
+    this.directories.clear();
+  }
+  getDirectory(locator) {
+    const directory = this.directories.get(locator);
+    if (!directory) throw new Error("External directory registration does not exist");
+    return directory;
+  }
+  getWritableDirectory(locator) {
+    const directory = this.getDirectory(locator);
+    if (directory.mode !== "readwrite") throw new Error("External directory registration is not writable");
+    return directory;
+  }
+  getOpenWriter(writer) {
+    nonnegativeSafeInteger(writer, "External directory writer");
+    const state = this.writers.get(writer);
+    if (!state || state.terminal) throw new Error("External directory writer is not open");
+    return state;
+  }
+  async cleanupWriter(state) {
+    this.directories.get(state.locator)?.files.delete(state.path);
+    try {
+      await state.descriptor?.close();
+    } finally {
+      state.descriptor = void 0;
+      if (state.temp) {
+        await unlink(state.temp).catch(() => void 0);
+      }
+      state.temp = void 0;
+    }
+  }
+  enqueue(state, operation, afterFailure = false) {
+    const result = state.chain.then(async () => {
+      if (!afterFailure && state.failure !== void 0) throw state.failure;
+      await operation();
+    });
+    state.chain = result.catch((error) => {
+      state.failure ??= error;
+    });
+    return result;
+  }
+};
+
 // src/node_worker.ts
 if (!parentPort) throw new Error("Node engine worker requires a parent port");
 var enginePort = parentPort;
 var options = workerData;
 var engineUrl = options?.engineUrl ?? fileURLToPath(new URL("./hyperdb-wasm.cjs", import.meta.url));
-var hostPath = resolve(options?.hostPath ?? process.cwd());
+var storagePath = resolve2(options?.storagePath ?? process.cwd());
 var transport = {
   postMessage: (message, transfer = []) => enginePort.postMessage(message, [...transfer]),
   onMessage: (listener) => enginePort.on("message", listener)
 };
-var compressedWasm = join(dirname(engineUrl), "hyperdb-wasm.wasm.br");
+var compressedWasm = join2(dirname(engineUrl), "hyperdb-wasm.wasm.br");
 var decompressedWasm = Readable.toWeb(createReadStream(compressedWasm).pipe(createBrotliDecompress()));
 var compiledWasm = await WebAssembly.compileStreaming(new Response(decompressedWasm, {
   headers: { "Content-Type": "application/wasm" }
@@ -344,18 +932,126 @@ var module = createRequire(import.meta.url)(engineUrl);
 delete configuredGlobal.HYPERDB_WASM_MODULE;
 configureWorker(module, transport, (runtime) => {
   if (!runtime.NODEFS) throw new Error("Node filesystem backend is unavailable");
+  const nodeFS = runtime.NODEFS;
   runtime.FS.mkdirTree("/host");
-  runtime.FS.mount(runtime.NODEFS, { root: hostPath }, "/host");
   runtime.FS.mkdirTree("/hyperdb-persistent");
-  runtime.FS.mount(runtime.NODEFS, { root: hostPath }, "/hyperdb-persistent");
+  runtime.FS.mount(nodeFS, { root: storagePath }, "/hyperdb-persistent");
   runtime.FS.mkdirTree("/hyperdb-persistent/hyperdb");
   runtime.FS.mkdirTree("/hyperdb-persistent/logs");
+  const nodeRuntimeFS = runtime.FS;
+  const shouldAccessHostPath = (path) => isAbsolute2(path) && !nodeRuntimeFS.analyzePath(path).exists;
+  runtime.hyperdbWasmShouldOpenHostFile = shouldAccessHostPath;
+  runtime.hyperdbWasmShouldListHostDirectory = shouldAccessHostPath;
+  runtime.externalDirectoryBackend = new NodeExternalDirectoryBackend();
+  const externalFiles = /* @__PURE__ */ new Map();
+  const externalPaths = /* @__PURE__ */ new Map();
+  let nextHostFileId = 1;
+  const openExternalFile = (path, locator) => {
+    const descriptor = openSync(path, "r");
+    try {
+      const stat2 = fstatSync(descriptor, { bigint: true });
+      if (!stat2.isFile()) throw new Error(`External file is not a regular file: ${path}`);
+      if (stat2.size > BigInt(Number.MAX_SAFE_INTEGER)) throw new Error("External file size must be a nonnegative safe integer");
+      const file = { descriptor, size: Number(stat2.size) };
+      externalFiles.set(locator, file);
+      return file;
+    } catch (error) {
+      closeSync(descriptor);
+      throw error;
+    }
+  };
+  runtime.externalFileBackend = {
+    register: (path) => {
+      const absolute = resolve2(path);
+      const locator = `hyperdb-node-file:${randomUUID2()}`;
+      const { size } = openExternalFile(absolute, locator);
+      return { name: basename2(absolute), locator, size };
+    },
+    registered: (path, locator) => {
+      if (!externalFiles.has(locator)) throw new Error("External file descriptor is unavailable");
+      externalPaths.set(path, locator);
+    },
+    removed: (path) => {
+      const locator = externalPaths.get(path) ?? path;
+      const file = externalFiles.get(locator);
+      if (file === void 0) return;
+      closeSync(file.descriptor);
+      externalFiles.delete(locator);
+      externalPaths.delete(path);
+    },
+    openHostFile: (path) => {
+      if (!isAbsolute2(path)) throw new Error("Host file path must be absolute");
+      const id = nextHostFileId++;
+      if (!Number.isSafeInteger(id)) throw new Error("Host file identifier limit reached");
+      const { size } = openExternalFile(path, `hyperdb-node-host-file:${id}`);
+      return { id, size };
+    },
+    closeHostFile: (id) => {
+      const locator = `hyperdb-node-host-file:${id}`;
+      const file = externalFiles.get(locator);
+      if (!file) return;
+      closeSync(file.descriptor);
+      externalFiles.delete(locator);
+    },
+    listHostDirectory: async (path) => {
+      if (!isAbsolute2(path)) throw new Error("Host directory path must be absolute");
+      const files = [];
+      const visit = async (directory, prefix) => {
+        const entries = await opendir2(directory);
+        for await (const entry of entries) {
+          if (!entry.name || entry.name === "." || entry.name === ".." || entry.name.includes("/") || entry.name.includes("\\") || entry.name.includes("\0")) {
+            throw new Error("Host directory contains an invalid entry name");
+          }
+          const relativePath = prefix ? `${prefix}/${entry.name}` : entry.name;
+          if (entry.isDirectory()) await visit(join2(directory, entry.name), relativePath);
+          else if (entry.isFile()) files.push(relativePath);
+          else if (entry.isSymbolicLink() && (await stat(join2(directory, entry.name))).isFile()) files.push(relativePath);
+        }
+      };
+      await visit(path, "");
+      return files;
+    },
+    read: (locator, offset, length, expectedSize) => {
+      if (!locator.startsWith("hyperdb-node-file:") && !locator.startsWith("hyperdb-node-host-file:")) return void 0;
+      return (async () => {
+        const file = externalFiles.get(locator);
+        if (file === void 0) throw new Error("External file registration does not exist");
+        const end = offset + length;
+        if (expectedSize !== file.size || !Number.isSafeInteger(offset) || offset < 0 || !Number.isSafeInteger(length) || length < 0 || !Number.isSafeInteger(end) || end > file.size) {
+          throw new Error("External file byte range is invalid");
+        }
+        const result = new Uint8Array(length);
+        let bytesRead = 0;
+        while (bytesRead < length) {
+          const count = await new Promise((resolveRead, rejectRead) => {
+            read(file.descriptor, result, bytesRead, length - bytesRead, offset + bytesRead, (error, readCount) => {
+              if (error) rejectRead(error);
+              else resolveRead(readCount);
+            });
+          });
+          if (count === 0) throw new Error("External file ended before the registered size");
+          bytesRead += count;
+        }
+        return result;
+      })();
+    },
+    shutdown: () => {
+      for (const { descriptor } of externalFiles.values()) closeSync(descriptor);
+      externalFiles.clear();
+      externalPaths.clear();
+    }
+  };
+  const hostDirectories = /* @__PURE__ */ new Map();
   runtime.resolveHyperDBPath = (path) => {
-    const absolute = resolve(isAbsolute(path) ? path : join(hostPath, path));
-    const suffix = relative(hostPath, absolute);
-    if (suffix === ".." || suffix.startsWith(`..${sep}`) || isAbsolute(suffix)) {
-      throw new Error(`Arrow IPC path is outside the mounted host directory: ${path}`);
+    const absolute = resolve2(path);
+    const directory = dirname(absolute);
+    let mount = hostDirectories.get(directory);
+    if (mount === void 0) {
+      mount = `/host/${hostDirectories.size}`;
+      runtime.FS.mkdirTree(mount);
+      runtime.FS.mount(nodeFS, { root: directory }, mount);
+      hostDirectories.set(directory, mount);
     }
-    return posix.join("/host", ...suffix.split(sep));
+    return posix.join(mount, basename2(absolute));
   };
 });