hyperdb-wasm 0.0.26862-dev.wasm.g89537caa → 0.0.27000-dev.wasm.g0bde3882

package/dist/browser_worker.js

@@ -16,6 +16,12 @@
     if (!value) throw new Error(`${name} must not be empty`);
     if (value.includes("\0")) throw new Error(`${name} must not contain NUL`);
   }
+  function resultError(result) {
+    if (result.state === "error" || result.state === "busy") {
+      return typeof result.error === "string" ? result.error : result.error.message;
+    }
+    return `unexpected Wasm result state: ${result.state}`;
+  }
   function decodeStructuredError(payload) {
     if (!structuredErrorMagic.every((byte, index) => payload[index] === byte)) throw new Error("structured error has invalid magic");
     const view = new DataView(payload.buffer, payload.byteOffset, payload.byteLength);
@@ -52,6 +58,8 @@
     registerExternalFile;
     registerExternalDirectory;
     removeExternalRegistration;
+    projectExternalFile;
+    projectExternalDirectory;
     startQuery;
     insertArrowIPCFromPath;
     pollQuery;
@@ -61,6 +69,7 @@
     resultPayloadLength;
     free;
     nextTemporaryFile = 1;
+    mountedPaths = /* @__PURE__ */ new Set();
     constructor(module) {
       this.module = module;
       this.initialize = module.cwrap("hyperdb_wasm_initialize", "pointer", ["string", "string"]);
@@ -70,6 +79,8 @@
       this.registerExternalFile = module.cwrap("hyperdb_wasm_register_external_file", "pointer", ["string", "string", "number"]);
       this.registerExternalDirectory = module.cwrap("hyperdb_wasm_register_external_directory", "pointer", ["string", "string", "number"]);
       this.removeExternalRegistration = module.cwrap("hyperdb_wasm_remove_external_registration", "pointer", ["string"]);
+      this.projectExternalFile = module.cwrap("hyperdb_wasm_project_external_file", "pointer", ["string", "string", "number"]);
+      this.projectExternalDirectory = module.cwrap("hyperdb_wasm_project_external_directory", "pointer", ["string", "string"]);
       this.startQuery = module.cwrap("hyperdb_wasm_query_start", "pointer", ["number", "string", "number"]);
       this.insertArrowIPCFromPath = module.cwrap("hyperdb_wasm_arrow_insert_from_path", "pointer", ["number", "string", "string", "string", "number"]);
       this.pollQuery = module.cwrap("hyperdb_wasm_query_poll", "pointer", ["number"]);
@@ -90,77 +101,40 @@
         this.module.FS.unlink(request.path);
         return { response: { id: request.id, type: "response", result: { state: "ok", payload: new Uint8Array() } }, transfer: [] };
       }
-      if (request.type === "registerExternalFile" && "path" in request) {
+      if (request.type === "mount" && request.source.kind === "node-path") {
         if (!this.module.externalFileBackend) {
           return { response: { id: request.id, type: "response", result: { state: "error", error: "Host external files are supported only by the Node client" } }, transfer: [] };
         }
-        const registration = this.module.externalFileBackend.register(request.path);
-        try {
-          const pointer2 = safeInteger(this.registerExternalFile(registration.name, registration.locator, registration.size), "result pointer");
-          const result2 = this.decodeResult(pointer2);
-          if (result2.state === "ok") {
-            const path = new TextDecoder("utf-8", { fatal: true }).decode(result2.payload);
-            try {
-              this.module.externalFileBackend.registered(path, registration.locator);
-            } catch (error) {
-              const cleanupPointer = safeInteger(this.removeExternalRegistration(path), "result pointer");
-              this.decodeResult(cleanupPointer);
-              throw error;
-            }
-          } else {
-            this.module.externalFileBackend.removed(registration.locator);
-          }
-          return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
-        } catch (error) {
-          this.module.externalFileBackend.removed(registration.locator);
-          throw error;
-        }
+        const registration = this.module.externalFileBackend.mount(request.source.path, request.mode);
+        if (registration.kind === "directory") return this.mountDirectory(request, request.source.path);
+        return this.mountFile(request, registration.locator, registration.size, this.module.externalFileBackend);
       }
-      if (request.type === "registerExternalFile" && !this.module.externalFileBackendAvailable) {
+      if (request.type === "mount" && request.source.kind === "browser-file" && !this.module.externalFileBackendAvailable) {
         return { response: { id: request.id, type: "response", result: { state: "error", error: "Blob external files are supported only by the browser client" } }, transfer: [] };
       }
-      if (request.type === "registerExternalDirectory") {
-        const backend = this.module.externalDirectoryBackend;
-        if (!backend) {
-          return { response: { id: request.id, type: "response", result: { state: "error", error: "External directories are not supported by this client" } }, transfer: [] };
-        }
-        const registration = backend.register("handle" in request ? request.handle : request.path, request.mode);
-        try {
-          const pointer2 = safeInteger(this.registerExternalDirectory(registration.name, registration.locator, Number(request.mode === "readwrite")), "result pointer");
-          const result2 = this.decodeResult(pointer2);
-          if (result2.state === "ok") {
-            const path = new TextDecoder("utf-8", { fatal: true }).decode(result2.payload);
-            try {
-              backend.registered(path, registration.locator);
-            } catch (error) {
-              const cleanupPointer = safeInteger(this.removeExternalRegistration(path), "result pointer");
-              this.decodeResult(cleanupPointer);
-              throw error;
-            }
-          } else {
-            const completion = completionFor([backend.removed(registration.locator)]);
-            return { response: { id: request.id, type: "response", result: result2 }, transfer: [], ...completion && { completion } };
-          }
-          return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
-        } catch (error) {
-          void Promise.resolve(backend.removed(registration.locator)).catch(() => void 0);
-          throw error;
-        }
+      if (request.type === "mount" && request.source.kind === "browser-directory") {
+        return this.mountDirectory(request, request.source.handle);
       }
-      if (request.type === "removeExternalRegistration" && (this.module.externalFileBackend || this.module.externalDirectoryBackend)) {
+      if (request.type === "mount" && request.source.kind === "browser-file") {
+        return this.mountFile(request, request.source.url, safeInteger(request.source.size, "external file size"));
+      }
+      if (request.type === "unmount" && (this.module.externalFileBackend || this.module.externalDirectoryBackend)) {
         validateCString(request.path, "external registration path");
         const pointer2 = safeInteger(this.removeExternalRegistration(request.path), "result pointer");
         const result2 = this.decodeResult(pointer2);
-        if (result2.state === "ok") {
-          const completion = completionFor([
-            this.module.externalFileBackend?.removed(request.path),
-            this.module.externalDirectoryBackend?.removed(request.path)
-          ]);
-          return { response: { id: request.id, type: "response", result: result2 }, transfer: [], ...completion && { completion } };
+        if (result2.state === "ok" || result2.state === "error" && result2.error === "External registration was not found" && this.mountedPaths.has(request.path)) {
+          const cleanup = async () => {
+            await Promise.all([
+              this.module.externalFileBackend?.removed(request.path),
+              this.module.externalDirectoryBackend?.removed(request.path)
+            ]);
+            this.mountedPaths.delete(request.path);
+          };
+          return { response: { id: request.id, type: "response", result: { state: "ok", payload: new Uint8Array() } }, transfer: [], completion: cleanup() };
         }
         return { response: { id: request.id, type: "response", result: result2 }, transfer: [] };
       }
-      if (request.type === "removeExternalRegistration" && !this.module.externalFileBackendAvailable) {
+      if (request.type === "unmount" && !this.module.externalFileBackendAvailable) {
         return { response: { id: request.id, type: "response", result: { state: "error", error: "External files are not supported by this client" } }, transfer: [] };
       }
       const pointer = safeInteger(this.invoke(request), "result pointer");
@@ -190,20 +164,15 @@
           return this.disconnect(request.connection);
         case "shutdown":
           return this.shutdown();
-        case "registerExternalFile":
-          if ("path" in request) throw new Error("host external file request has no backend");
-          validateCString(request.name, "external file name");
-          validateCString(request.url, "external file URL");
-          return this.registerExternalFile(request.name, request.url, safeInteger(request.size, "external file size"));
-        case "registerExternalDirectory":
-          throw new Error("external directory request has no backend");
-        case "removeExternalRegistration":
+        case "mount":
+          throw new Error("mount request has no backend");
+        case "unmount":
           validateCString(request.path, "external registration path");
           return this.removeExternalRegistration(request.path);
         case "startQuery":
           return this.startQuery(request.connection, request.sql, Number(request.sqlTestMode ?? false));
         case "insertArrowIPCFromPath": {
-          const path = request.internal ? request.path : this.module.resolveHyperDBPath?.(request.path) ?? request.path;
+          const path = request.internal || request.path.startsWith("/mnt/external/") ? request.path : this.module.resolveHyperDBPath?.(request.path) ?? request.path;
           return this.insertArrowIPCFromPath(request.connection, path, request.name, request.schema ?? "", Number(request.create));
         }
         case "pollQuery":
@@ -214,6 +183,88 @@
           return this.finishQuery(request.query);
       }
     }
+    mountFile(request, locator, size, backend) {
+      validateCString(request.path, "external registration path");
+      validateCString(locator, "external file locator");
+      let path;
+      let registered = false;
+      let rollbackError;
+      try {
+        const result = this.decodeResult(safeInteger(this.registerExternalFile(request.path, locator, size), "result pointer"));
+        if (result.state !== "ok") {
+          backend?.removed(locator);
+          return { response: { id: request.id, type: "response", result }, transfer: [] };
+        }
+        path = new TextDecoder("utf-8", { fatal: true }).decode(result.payload);
+        const projection = this.decodeResult(safeInteger(this.projectExternalFile(path, locator, size), "result pointer"));
+        if (projection.state !== "ok") throw new Error(resultError(projection));
+        backend?.registered(path, locator);
+        registered = true;
+        this.mountedPaths.add(path);
+        return { response: { id: request.id, type: "response", result }, transfer: [] };
+      } catch (error) {
+        if (path !== void 0) {
+          try {
+            const rollback = this.decodeResult(safeInteger(this.removeExternalRegistration(path), "result pointer"));
+            if (rollback.state !== "ok") rollbackError = new Error(resultError(rollback));
+          } catch (failure) {
+            rollbackError = failure;
+          }
+        }
+        if (rollbackError !== void 0) {
+          if (path !== void 0) {
+            backend?.registered(path, locator);
+            this.mountedPaths.add(path);
+          }
+          throw new Error(`mount failed and rollback also failed: ${rollbackError instanceof Error ? rollbackError.message : String(rollbackError)}`, { cause: error });
+        }
+        backend?.removed(registered && path !== void 0 ? path : locator);
+        throw error;
+      }
+    }
+    mountDirectory(request, source) {
+      const backend = this.module.externalDirectoryBackend;
+      if (!backend) {
+        return { response: { id: request.id, type: "response", result: { state: "error", error: "External directories are not supported by this client" } }, transfer: [] };
+      }
+      const registration = backend.register(source, request.mode);
+      let preserveBackend = false;
+      try {
+        const pointer = safeInteger(this.registerExternalDirectory(request.path, registration.locator, Number(request.mode === "readwrite")), "result pointer");
+        const result = this.decodeResult(pointer);
+        if (result.state === "ok") {
+          const path = new TextDecoder("utf-8", { fatal: true }).decode(result.payload);
+          try {
+            const projectionPointer = this.projectExternalDirectory(path, registration.locator);
+            const projection = this.decodeResult(safeInteger(projectionPointer, "result pointer"));
+            if (projection.state !== "ok") throw new Error(resultError(projection));
+            backend.registered(path, registration.locator);
+            this.mountedPaths.add(path);
+          } catch (error) {
+            const cleanupPointer = safeInteger(this.removeExternalRegistration(path), "result pointer");
+            const cleanup = this.decodeResult(cleanupPointer);
+            if (cleanup.state !== "ok") {
+              backend.registered(path, registration.locator);
+              this.mountedPaths.add(path);
+              preserveBackend = true;
+              throw new Error(`mount failed and rollback also failed: ${resultError(cleanup)}`, { cause: error });
+            }
+            throw error;
+          }
+        } else {
+          const completion = completionFor([backend.removed(registration.locator)]);
+          return { response: { id: request.id, type: "response", result }, transfer: [], ...completion && { completion } };
+        }
+        return { response: { id: request.id, type: "response", result }, transfer: [] };
+      } catch (error) {
+        if (preserveBackend) throw error;
+        return {
+          response: { id: request.id, type: "response", result: { state: "error", error: error instanceof Error ? error.message : String(error) } },
+          transfer: [],
+          completion: Promise.resolve(backend.removed(registration.locator))
+        };
+      }
+    }
     decodeResult(pointer) {
       if (pointer === 0) return { state: "error", error: "out of memory" };
       try {
@@ -278,12 +329,21 @@
         return hasKeys(value, ["id", "type", "bytes"]) && value.bytes instanceof Uint8Array;
       case "removeFile":
         return hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
-      case "registerExternalFile":
-        return hasKeys(value, ["id", "type", "name", "url", "size"]) && typeof value.name === "string" && typeof value.url === "string" && typeof value.size === "number" && Number.isSafeInteger(value.size) && value.size >= 0 || hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
-      case "registerExternalDirectory":
-        return (value.mode === "read" || value.mode === "readwrite") && (hasKeys(value, ["id", "type", "handle", "mode"]) && isRecord(value.handle) && value.handle.kind === "directory" || hasKeys(value, ["id", "type", "path", "mode"]) && typeof value.path === "string" && !!value.path && !value.path.includes("\0"));
-      case "removeExternalRegistration":
-        return hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string";
+      case "mount": {
+        if (!hasKeys(value, ["id", "type", "source", "path", "mode"]) || typeof value.path !== "string" || !value.path.startsWith("/mnt/external/") || new TextEncoder().encode(value.path.slice("/mnt/external/".length)).byteLength === 0 || new TextEncoder().encode(value.path.slice("/mnt/external/".length)).byteLength > 200 || value.path.slice("/mnt/external/".length).split("/").some((component) => !component || component === "." || component === ".." || !/^[A-Za-z0-9._-]+$/.test(component)) || value.mode !== "read" && value.mode !== "readwrite" || !isRecord(value.source)) return false;
+        switch (value.source.kind) {
+          case "browser-file":
+            return value.mode === "read" && hasKeys(value.source, ["kind", "name", "url", "size"]) && typeof value.source.name === "string" && typeof value.source.url === "string" && typeof value.source.size === "number" && Number.isSafeInteger(value.source.size) && value.source.size >= 0;
+          case "browser-directory":
+            return hasKeys(value.source, ["kind", "handle"]) && isRecord(value.source.handle) && value.source.handle.kind === "directory";
+          case "node-path":
+            return hasKeys(value.source, ["kind", "path"]) && typeof value.source.path === "string" && !!value.source.path && !value.source.path.includes("\0");
+          default:
+            return false;
+        }
+      }
+      case "unmount":
+        return hasKeys(value, ["id", "type", "path"]) && typeof value.path === "string" && !!value.path && !value.path.includes("\0");
       case "pollQuery":
       case "cancelQuery":
       case "finishQuery":
@@ -371,33 +431,6 @@
         module._hyperdb_wasm_fetch_complete(id, 0, -1);
       });
     };
-    module.hyperdbWasmHttpStart = (id, method, url, headerLines, bodyPointer, bodyLength) => {
-      const headers = new Headers();
-      for (const line of headerLines.split("\n")) {
-        const separator = line.indexOf(":");
-        if (separator > 0) headers.set(line.slice(0, separator), line.slice(separator + 1));
-      }
-      const pointer = safePointer(bodyPointer, "HTTP body pointer");
-      const bodyEnd = safePointer(pointer + bodyLength, "HTTP body end");
-      if (bodyEnd > module.HEAPU8.byteLength) throw new Error("HTTP body exceeds the wasm heap");
-      const body = bodyLength > 0 ? module.HEAPU8.slice(pointer, bodyEnd) : void 0;
-      const init = { method, headers };
-      if (body) init.body = body;
-      void fetch(url, init).then(async (response) => {
-        const responseBody = new Uint8Array(await response.arrayBuffer());
-        let responseHeaders = "";
-        response.headers.forEach((value, name) => {
-          responseHeaders += `${name.toLowerCase()}:${value}
-`;
-        });
-        const headersPointer = responseHeaders ? safePointer(module.stringToNewUTF8(responseHeaders), "headers pointer") : 0;
-        const responseBodyPointer = responseBody.byteLength ? allocate(module, responseBody) : 0;
-        module._hyperdb_wasm_http_complete(id, response.status, headersPointer, responseBodyPointer, responseBody.byteLength);
-      }).catch((error) => {
-        const errorBytes = new TextEncoder().encode(errorText(error));
-        module._hyperdb_wasm_http_complete(id, -1, 0, allocate(module, errorBytes), errorBytes.byteLength);
-      });
-    };
   }
   function installExternalDirectoryRuntime(module) {
     const backend = module.externalDirectoryBackend;
@@ -490,6 +523,54 @@
       };
     }
   }
+  function installFilesystemProjectionRuntime(module) {
+    const complete = module._hyperdb_wasm_filesystem_operation_complete;
+    if (!complete) return;
+    module.hyperdbWasmFilesystemListStart = (id, locator, path) => {
+      const listing = module.externalDirectoryBackend?.listEntries?.(locator, path);
+      if (!listing) {
+        complete(id, 0, -1);
+        return;
+      }
+      void listing.then((entries) => {
+        const encoded = entries.map((entry) => `${entry.kind === "file" ? "f" : "d"}:${entry.name}`);
+        complete(id, safePointer(module.stringToNewUTF8(JSON.stringify(encoded)), "filesystem listing pointer"), 0);
+      }).catch(() => complete(id, 0, -1));
+    };
+    module.hyperdbWasmFilesystemLookupStart = (id, locator, path) => {
+      const lookup = module.externalDirectoryBackend?.lookup?.(locator, path);
+      if (!lookup) {
+        complete(id, 0, -1);
+        return;
+      }
+      void lookup.then((entry) => {
+        if (entry === void 0) complete(id, 0, -2);
+        else if (entry.kind === "directory") complete(id, 0, -21);
+        else complete(id, safePointer(module.stringToNewUTF8(entry.locator), "filesystem file locator pointer"), entry.size);
+      }).catch(() => complete(id, 0, -1));
+    };
+    module.hyperdbWasmFilesystemReadStart = (id, locator, offset, length, expectedSize, pointerValue) => {
+      const read = module.externalFileBackend?.read(locator, offset, length, expectedSize) ?? module.externalDirectoryBackend?.read(locator, offset, length, expectedSize) ?? (locator.startsWith("blob:") ? fetch(locator, { headers: { Range: `bytes=${offset}-${offset + length - 1}` } }).then(async (response) => {
+        const contentRange = /^bytes (\d+)-(\d+)\/(\d+)$/.exec(response.headers.get("content-range") ?? "");
+        if (response.status !== 206 || !contentRange || contentRange[1] !== String(offset) || contentRange[2] !== String(offset + length - 1) || contentRange[3] !== String(expectedSize)) {
+          throw new Error("Blob URL returned an invalid byte range");
+        }
+        return new Uint8Array(await response.arrayBuffer());
+      }) : void 0);
+      if (!read) {
+        complete(id, 0, -1);
+        return;
+      }
+      void read.then((bytes) => {
+        const pointer = safePointer(pointerValue, "filesystem read pointer");
+        const end = safePointer(pointer + bytes.byteLength, "filesystem read end");
+        module.getValue(pointer, "i8");
+        if (bytes.byteLength !== length || end > module.HEAPU8.byteLength) throw new Error("filesystem read returned an invalid payload");
+        module.HEAPU8.set(bytes, pointer);
+        complete(id, 0, bytes.byteLength);
+      }).catch(() => complete(id, 0, -1));
+    };
+  }
   function configureWorker(module, transport, configureFilesystem = () => void 0) {
     const queuedRequests = [];
     let bindings;
@@ -523,6 +604,7 @@
         installHttpRuntime(module, transport);
         installHostFileRuntime(module);
         installExternalDirectoryRuntime(module);
+        installFilesystemProjectionRuntime(module);
         bindings = new HyperDBBindings(module);
         transport.postMessage({ type: "ready" });
         for (const request of queuedRequests.splice(0)) handle(request);
@@ -608,6 +690,32 @@
       files.sort();
       return files;
     }
+    async listEntries(locator, path) {
+      const handle = await this.openDirectory(this.getDirectory(locator), directorySegments(path));
+      const entries = [];
+      for await (const child of handle.values()) {
+        validateSegment(child.name);
+        if (child.kind !== "file" && child.kind !== "directory") throw new Error("External directory contains an unsupported entry");
+        entries.push({ name: child.name, kind: child.kind });
+      }
+      entries.sort((left, right) => left.name.localeCompare(right.name));
+      return entries;
+    }
+    async lookup(locator, path) {
+      const directory = this.getDirectory(locator);
+      const segments = relativeSegments(path);
+      const parent = await this.openDirectory(directory, segments.slice(0, -1));
+      const name = segments.at(-1);
+      if (!name) return { kind: "directory" };
+      for await (const child of parent.values()) {
+        validateSegment(child.name);
+        if (child.name !== name) continue;
+        if (child.kind === "directory") return { kind: "directory" };
+        if (child.kind === "file") return { kind: "file", ...await this.open(locator, path) };
+        throw new Error("External directory contains an unsupported entry");
+      }
+      return void 0;
+    }
     async open(locator, path) {
       const directory = this.getDirectory(locator);
       const cached = directory.files.get(path);
@@ -757,6 +865,15 @@
       if (directory.mode !== "readwrite") throw new Error("External directory registration is not writable");
       return directory;
     }
+    async openDirectory(directory, segments) {
+      let handle = directory.handle;
+      for (const segment of segments) {
+        const child = await handle.getDirectoryHandle(segment);
+        if (child.kind !== "directory" || child.name !== segment) throw new Error("External directory returned an unexpected child handle");
+        handle = child;
+      }
+      return handle;
+    }
     getOpenWriter(writer) {
       nonnegativeSafeInteger(writer, "External directory writer");
       const state = this.writers.get(writer);

package/dist/client.d.ts

@@ -1,4 +1,5 @@
-import type { SetupProgress, WasmResult, WorkerTransport } from './protocol.js';
+import type { MountAccessMode } from './api.js';
+import type { MountSource, SetupProgress, WasmResult, WorkerTransport } from './protocol.js';
 export declare class HyperDBEngineClient {
     private readonly onSetupProgress?;
     private readonly transport;
@@ -19,10 +20,8 @@ export declare class HyperDBEngineClient {
     insertArrowIPCFromPath(connection: number, path: string, name: string, schema: string | null, create: boolean, internal?: boolean): Promise<WasmResult>;
     createTemporaryFile(bytes: Uint8Array): Promise<WasmResult>;
     removeFile(path: string): Promise<WasmResult>;
-    registerExternalFile(path: string): Promise<WasmResult>;
-    registerExternalFile(name: string, url: string, size: number): Promise<WasmResult>;
-    registerExternalDirectory(source: unknown, mode: 'read' | 'readwrite'): Promise<WasmResult>;
-    removeExternalRegistration(path: string): Promise<WasmResult>;
+    mount(source: MountSource, path: string, mode: MountAccessMode): Promise<WasmResult>;
+    unmount(path: string): Promise<WasmResult>;
     pollQuery(query: number): Promise<WasmResult>;
     cancelQuery(query: number): Promise<WasmResult>;
     finishQuery(query: number): Promise<WasmResult>;