hyperdb-wasm 0.0.26268-dev.remotes-origin-wasm.gcf8d815a

package/dist/raw.node.js

@@ -0,0 +1,281 @@
+// protocol.ts
+function isRecord(value) {
+  return typeof value === "object" && value !== null;
+}
+function hasKeys(value, keys) {
+  const actual = Object.keys(value).sort();
+  const expected = [...keys].sort();
+  return actual.length === expected.length && actual.every((key, index) => key === expected[index]);
+}
+function isWasmResult(value) {
+  if (!isRecord(value) || typeof value.state !== "string") return false;
+  switch (value.state) {
+    case "ok":
+    case "chunk":
+      return hasKeys(value, ["state", "payload"]) && value.payload instanceof Uint8Array;
+    case "pending":
+    case "done":
+      return hasKeys(value, ["state"]);
+    case "error":
+    case "busy":
+      return hasKeys(value, ["state", "error"]) && typeof value.error === "string";
+    default:
+      return false;
+  }
+}
+function isWorkerEvent(message) {
+  if (!isRecord(message) || typeof message.type !== "string") return false;
+  switch (message.type) {
+    case "ready":
+      return hasKeys(message, ["type"]);
+    case "fatal":
+      return hasKeys(message, ["type", "error"]) && typeof message.error === "string";
+    case "log":
+      return hasKeys(message, ["type", "text"]) && typeof message.text === "string";
+    case "response":
+      return hasKeys(message, ["id", "type", "result"]) && Number.isSafeInteger(message.id) && isWasmResult(message.result);
+    default:
+      return false;
+  }
+}
+
+// client.ts
+var HyperDBEngineClient = class {
+  transport;
+  pending = /* @__PURE__ */ new Map();
+  readyPromise;
+  resolveReady;
+  rejectReady;
+  nextId = 1;
+  failed;
+  unsubscribe;
+  constructor(transport) {
+    this.transport = transport;
+    this.readyPromise = new Promise((resolve, reject) => {
+      this.resolveReady = resolve;
+      this.rejectReady = reject;
+    });
+    this.unsubscribe = [
+      transport.onMessage((message) => this.onMessage(message)),
+      transport.onError((error) => this.fail(error)),
+      transport.onClose((error) => this.fail(error))
+    ];
+  }
+  ready() {
+    return this.readyPromise;
+  }
+  connect() {
+    return this.request({ id: this.nextId++, type: "connect" });
+  }
+  disconnect(connection) {
+    return this.request({ id: this.nextId++, type: "disconnect", connection });
+  }
+  startQuery(connection, sql) {
+    return this.request({ id: this.nextId++, type: "startQuery", connection, sql });
+  }
+  insertArrowIPCFromPath(connection, path, name, schema, create, internal = false) {
+    return this.request({ id: this.nextId++, type: "insertArrowIPCFromPath", connection, path, name, schema, create, internal });
+  }
+  createTemporaryFile(bytes) {
+    const payload = bytes.slice();
+    return this.request({ id: this.nextId++, type: "createTemporaryFile", bytes: payload }, [payload.buffer]);
+  }
+  removeFile(path) {
+    return this.request({ id: this.nextId++, type: "removeFile", path });
+  }
+  pollQuery(query) {
+    return this.request({ id: this.nextId++, type: "pollQuery", query });
+  }
+  cancelQuery(query) {
+    return this.request({ id: this.nextId++, type: "cancelQuery", query });
+  }
+  releaseQuery(query) {
+    return this.request({ id: this.nextId++, type: "releaseQuery", query });
+  }
+  async terminate() {
+    this.fail(new Error("engine worker terminated"));
+    await this.transport.terminate();
+  }
+  request(request, transfer = []) {
+    if (this.failed) return Promise.reject(this.failed);
+    return new Promise((resolve, reject) => {
+      this.pending.set(request.id, { resolve, reject });
+      try {
+        this.transport.postMessage(request, transfer);
+      } catch (error) {
+        this.pending.delete(request.id);
+        reject(error instanceof Error ? error : new Error(String(error)));
+      }
+    });
+  }
+  onMessage(message) {
+    if (!isWorkerEvent(message)) return;
+    if (message.type === "ready") {
+      this.resolveReady();
+      return;
+    }
+    if (message.type === "fatal") {
+      this.fail(new Error(message.error || "engine aborted"));
+      return;
+    }
+    if (message.type !== "response") return;
+    const pending = this.pending.get(message.id);
+    if (!pending) return;
+    this.pending.delete(message.id);
+    pending.resolve(message.result);
+  }
+  fail(error) {
+    if (this.failed) return;
+    this.failed = error;
+    for (const unsubscribe of this.unsubscribe.splice(0)) unsubscribe();
+    this.rejectReady(error);
+    for (const pending of this.pending.values()) pending.reject(error);
+    this.pending.clear();
+  }
+};
+
+// wasm_bindings.ts
+var resultStates = ["ok", "pending", "chunk", "done", "error", "busy"];
+function safeInteger(value, name) {
+  const number = typeof value === "bigint" && value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : value;
+  if (typeof number !== "number" || !Number.isSafeInteger(number) || number < 0) throw new Error(`${name} must be a nonnegative safe integer`);
+  return number;
+}
+var HyperDBBindings = class {
+  module;
+  connect;
+  disconnect;
+  startQuery;
+  insertArrowIPCFromPath;
+  pollQuery;
+  cancelQuery;
+  releaseQuery;
+  resultState;
+  resultPayloadLength;
+  free;
+  nextTemporaryFile = 1;
+  constructor(module) {
+    this.module = module;
+    this.connect = module.cwrap("hyperdb_wasm_connect", "pointer", []);
+    this.disconnect = module.cwrap("hyperdb_wasm_disconnect", "pointer", ["number"]);
+    this.startQuery = module.cwrap("hyperdb_wasm_query_start", "pointer", ["number", "string"]);
+    this.insertArrowIPCFromPath = module.cwrap("hyperdb_wasm_arrow_insert_from_path", "pointer", ["number", "string", "string", "string", "number"]);
+    this.pollQuery = module.cwrap("hyperdb_wasm_query_poll", "pointer", ["number"]);
+    this.cancelQuery = module.cwrap("hyperdb_wasm_query_cancel", "pointer", ["number"]);
+    this.releaseQuery = module.cwrap("hyperdb_wasm_query_release", "pointer", ["number"]);
+    this.resultState = module.cwrap("hyperdb_wasm_result_state", "number", ["number"]);
+    this.resultPayloadLength = module.cwrap("hyperdb_wasm_result_payload_len", "number", ["number"]);
+    this.free = module.cwrap("hyperdb_wasm_free", null, ["pointer"]);
+  }
+  handle(request) {
+    if (request.type === "createTemporaryFile") {
+      const path = `/tmp/hyperdb/arrow-${this.nextTemporaryFile++}.arrows`;
+      this.module.FS.mkdirTree("/tmp/hyperdb");
+      this.module.FS.writeFile(path, request.bytes);
+      return { response: { id: request.id, type: "response", result: { state: "ok", payload: new TextEncoder().encode(path) } }, transfer: [] };
+    }
+    if (request.type === "removeFile") {
+      this.module.FS.unlink(request.path);
+      return { response: { id: request.id, type: "response", result: { state: "ok", payload: new Uint8Array() } }, transfer: [] };
+    }
+    const pointer = safeInteger(this.invoke(request), "result pointer");
+    const result = this.decodeResult(pointer);
+    const response = { id: request.id, type: "response", result };
+    return { response, transfer: result.state === "chunk" ? [result.payload.buffer] : [] };
+  }
+  invoke(request) {
+    switch (request.type) {
+      case "connect":
+        return this.connect();
+      case "disconnect":
+        return this.disconnect(request.connection);
+      case "startQuery":
+        return this.startQuery(request.connection, request.sql);
+      case "insertArrowIPCFromPath": {
+        const path = request.internal ? request.path : this.module.resolveHyperDBPath?.(request.path) ?? request.path;
+        return this.insertArrowIPCFromPath(request.connection, path, request.name, request.schema ?? "", Number(request.create));
+      }
+      case "pollQuery":
+        return this.pollQuery(request.query);
+      case "cancelQuery":
+        return this.cancelQuery(request.query);
+      case "releaseQuery":
+        return this.releaseQuery(request.query);
+    }
+  }
+  decodeResult(pointer) {
+    if (pointer === 0) return { state: "error", error: "out of memory" };
+    try {
+      const rawState = safeInteger(this.resultState(pointer), "result state");
+      const state = resultStates[rawState];
+      if (!state) return { state: "error", error: `invalid result state ${rawState}` };
+      const payloadLength = safeInteger(this.resultPayloadLength(pointer), "result payload length");
+      const payloadStart = safeInteger(pointer + 8, "result payload start");
+      const payloadEnd = safeInteger(payloadStart + payloadLength, "result payload end");
+      if (payloadEnd > this.module.HEAPU8.byteLength) throw new Error("result payload exceeds the wasm heap");
+      const payload = new Uint8Array(payloadLength);
+      payload.set(this.module.HEAPU8.subarray(payloadStart, payloadEnd));
+      switch (state) {
+        case "ok":
+          return { state, payload };
+        case "pending":
+          return { state };
+        case "chunk":
+          return { state, payload };
+        case "done":
+          return { state };
+        case "error":
+          return { state, error: new TextDecoder().decode(payload) };
+        case "busy":
+          return { state, error: new TextDecoder().decode(payload) };
+      }
+    } finally {
+      this.free(pointer);
+    }
+  }
+};
+
+// node.ts
+import { Worker as NativeWorker } from "node:worker_threads";
+import { fileURLToPath } from "node:url";
+var NodeWorker = class {
+  constructor(worker) {
+    this.worker = worker;
+  }
+  worker;
+  postMessage(message, transfer = []) {
+    this.worker.postMessage(message, transfer);
+  }
+  onMessage(listener) {
+    const wrapped = (message) => listener(message);
+    this.worker.on("message", wrapped);
+    return () => this.worker.off("message", wrapped);
+  }
+  onError(listener) {
+    const wrapped = (error) => listener(error instanceof Error ? error : new Error(String(error)));
+    this.worker.on("error", wrapped);
+    return () => this.worker.off("error", wrapped);
+  }
+  onClose(listener) {
+    const wrapped = (code) => listener(new Error(`engine worker exited with code ${String(code)}`));
+    this.worker.on("exit", wrapped);
+    return () => this.worker.off("exit", wrapped);
+  }
+  terminate() {
+    return this.worker.terminate();
+  }
+};
+function resolveNodeEnginePath(engineUrl) {
+  return fileURLToPath(engineUrl);
+}
+function createNodeClient(options = {}) {
+  const workerUrl = options.workerUrl ?? new URL("./node_worker.js", import.meta.url);
+  const engineUrl = options.engineUrl ?? resolveNodeEnginePath(new URL("./hyperdb-wasm.cjs", import.meta.url));
+  return new HyperDBEngineClient(new NodeWorker(new NativeWorker(workerUrl, { workerData: { engineUrl, hostPath: options.hostPath } })));
+}
+export {
+  HyperDBBindings,
+  HyperDBEngineClient,
+  NodeWorker,
+  createNodeClient
+};

package/dist/wasm_bindings.d.ts

@@ -0,0 +1,37 @@
+import type { EngineRequest, WasmResponse } from './protocol.js';
+type CFunction = (...args: readonly unknown[]) => unknown;
+export interface HyperDBWasmModule {
+    readonly HEAPU8: Uint8Array;
+    readonly FS: {
+        mkdirTree(path: string): void;
+        mount(type: unknown, options: Record<string, unknown>, path: string): void;
+        writeFile(path: string, bytes: Uint8Array): void;
+        unlink(path: string): void;
+    };
+    resolveHyperDBPath?(path: string): string;
+    _malloc(size: number): number | bigint;
+    cwrap(name: string, returnType?: string | null, argumentTypes?: readonly string[]): CFunction;
+}
+export type BindingResponse = {
+    response: WasmResponse;
+    transfer: readonly ArrayBuffer[];
+};
+export declare class HyperDBBindings {
+    private readonly module;
+    private readonly connect;
+    private readonly disconnect;
+    private readonly startQuery;
+    private readonly insertArrowIPCFromPath;
+    private readonly pollQuery;
+    private readonly cancelQuery;
+    private readonly releaseQuery;
+    private readonly resultState;
+    private readonly resultPayloadLength;
+    private readonly free;
+    private nextTemporaryFile;
+    constructor(module: HyperDBWasmModule);
+    handle(request: EngineRequest): BindingResponse;
+    private invoke;
+    private decodeResult;
+}
+export {};

package/dist/worker_setup.d.ts

@@ -0,0 +1,25 @@
+import type { HyperDBWasmModule } from './wasm_bindings.js';
+import type { WorkerEvent } from './protocol.js';
+export interface WorkerHostTransport {
+    postMessage(message: WorkerEvent, transfer?: readonly ArrayBuffer[]): void;
+    onMessage(listener: (message: unknown) => void): void;
+}
+export interface HyperDBWasmRuntimeModule extends HyperDBWasmModule {
+    HEAPU8: Uint8Array;
+    _malloc(size: number): number | bigint;
+    _hyperdb_wasm_fetch_complete(id: number, pointer: number, length: number): void;
+    _hyperdb_wasm_http_complete(id: number, status: number, headersPointer: number, bodyPointer: number, bodyLength: number): void;
+    stringToNewUTF8(value: string): number | bigint;
+    NODEFS?: unknown;
+    OPFS?: unknown;
+    resolveHyperDBPath?: (path: string) => string;
+    arguments?: string[];
+    print?: (text: string) => void;
+    printErr?: (text: string) => void;
+    onRuntimeInitialized?: () => void;
+    onAbort?: (reason: unknown) => void;
+    locateFile?: (path: string, prefix: string) => string;
+    hyperdbWasmFetchStart?: (id: number, url: string, offset: number, length: number) => void;
+    hyperdbWasmHttpStart?: (id: number, method: string, url: string, headers: string, bodyPointer: number | bigint, bodyLength: number) => void;
+}
+export declare function configureWorker(module: HyperDBWasmRuntimeModule, transport: WorkerHostTransport, configureFilesystem?: (module: HyperDBWasmRuntimeModule) => void): void;

package/examples/browser.html

@@ -0,0 +1,84 @@
+<!doctype html>
+<html lang="en">
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>HyperDB WebAssembly Example</title>
+    <style>
+        :root { color-scheme: light dark; font-family: ui-monospace, SFMono-Regular, Consolas, monospace; }
+        body { margin: 0 auto; max-width: 960px; padding: 2rem; }
+        textarea { box-sizing: border-box; min-height: 10rem; padding: 1rem; resize: vertical; width: 100%; }
+        button { cursor: pointer; margin: 1rem 0; padding: 0.6rem 1rem; }
+        table { border-collapse: collapse; width: 100%; }
+        th, td { border: 1px solid currentColor; padding: 0.5rem; text-align: left; }
+        #status { min-height: 1.5rem; }
+    </style>
+</head>
+<body>
+    <h1>HyperDB in the browser</h1>
+    <p>Edit the SQL and run it locally in the WebAssembly worker.</p>
+    <textarea id="sql">select sensor_id, count(*) as samples, avg(reading) as average_reading
+from (values
+    (101, 18.5),
+    (101, 19.5),
+    (202, 22.0)
+) as telemetry(sensor_id, reading)
+group by sensor_id
+order by sensor_id</textarea>
+    <button id="run" type="button">Run query</button>
+    <p id="status">Loading HyperDB...</p>
+    <div id="result"></div>
+
+    <script type="module">
+        import { HyperDB } from '../dist/browser.js';
+
+        const sql = document.querySelector('#sql');
+        const run = document.querySelector('#run');
+        const status = document.querySelector('#status');
+        const result = document.querySelector('#result');
+
+        const database = await HyperDB.create();
+        const connection = await database.connect();
+        status.textContent = 'Ready';
+
+        function render(table) {
+            const names = table.schema.fields.map(({ name }) => name);
+            const rows = Array.from({ length: table.numRows }, (_, row) =>
+                names.map((name) => String(table.getChild(name)?.get(row) ?? 'NULL')),
+            );
+            const tableElement = document.createElement('table');
+            const header = tableElement.createTHead().insertRow();
+            for (const name of names) {
+                const cell = document.createElement('th');
+                cell.textContent = name;
+                header.append(cell);
+            }
+            const body = tableElement.createTBody();
+            for (const values of rows) {
+                const row = body.insertRow();
+                for (const value of values) row.insertCell().textContent = value;
+            }
+            result.replaceChildren(tableElement);
+        }
+
+        run.addEventListener('click', async () => {
+            run.disabled = true;
+            status.textContent = 'Running...';
+            try {
+                const table = await connection.query(sql.value);
+                render(table);
+                status.textContent = `${table.numRows} row(s)`;
+            } catch (error) {
+                result.textContent = error instanceof Error ? error.message : String(error);
+                status.textContent = 'Query failed';
+            } finally {
+                run.disabled = false;
+            }
+        });
+
+        window.addEventListener('pagehide', () => {
+            void connection.close().then(() => database.terminate());
+        });
+    </script>
+</body>
+</html>

package/examples/node.mjs

@@ -0,0 +1,68 @@
+import { HyperDB } from 'hyperdb-wasm';
+import { makeTable, tableToIPC } from 'apache-arrow';
+import { mkdtemp, rm, writeFile } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+function rows(table) {
+    return Array.from({ length: table.numRows }, (_, row) => Object.fromEntries(
+        table.schema.fields.map(({ name }) => [name, table.getChild(name)?.get(row)]),
+    ));
+}
+
+const temporaryDirectory = await mkdtemp(join(tmpdir(), 'hyperdb-wasm-'));
+const database = await HyperDB.create({ hostPath: temporaryDirectory });
+const connection = await database.connect();
+
+try {
+    const expressions = await connection.query(`
+        select
+            6 * 7 as answer,
+            upper('query Hyper from Node.js') as message
+    `);
+    console.table(rows(expressions));
+
+    const grouped = await connection.query(`
+        select sensor_id, count(*) as samples, min(reading) as minimum, max(reading) as maximum
+        from (values
+            (101, 18),
+            (101, 19),
+            (202, 22)
+        ) as readings(sensor_id, reading)
+        group by sensor_id
+        order by sensor_id
+    `);
+    console.table(rows(grouped));
+
+    const reader = await connection.send(`
+        select value, value * value as square
+        from generate_series(1, 5) as numbers(value)
+        order by value
+    `);
+    for await (const batch of reader) {
+        console.log(`streamed Arrow batch with ${batch.numRows} rows`);
+    }
+
+    const memoryReadings = makeTable({
+        sensor_id: new Int32Array([303, 404]),
+        reading: new Int32Array([24, 27]),
+    });
+    await connection.insertArrowTable(memoryReadings, { name: 'memory_readings' });
+
+    const temporaryReadings = makeTable({
+        sensor_id: new Int32Array([505, 606]),
+        reading: new Int32Array([29, 31]),
+    });
+    const readingsPath = join(temporaryDirectory, 'arrow-readings.arrows');
+    await writeFile(readingsPath, tableToIPC(temporaryReadings, 'stream'));
+    await connection.insertArrowIPCFromPath(readingsPath, { name: 'temporary_readings' });
+
+    console.log('Arrow IPC staged in the in-memory WasmFS:');
+    console.table(rows(await connection.query('select * from memory_readings order by sensor_id')));
+    console.log(`Arrow IPC read from the Node temporary file ${readingsPath}:`);
+    console.table(rows(await connection.query('select * from temporary_readings order by sensor_id')));
+} finally {
+    await connection.close();
+    await database.terminate();
+    await rm(temporaryDirectory, { recursive: true, force: true });
+}

package/examples/serve.mjs

@@ -0,0 +1,35 @@
+import { createReadStream, existsSync, statSync } from 'node:fs';
+import { createServer } from 'node:http';
+import { extname, resolve, sep } from 'node:path';
+
+const packageRoot = resolve(import.meta.dirname, '..');
+const port = Number(process.env.PORT ?? 8000);
+const contentTypes = {
+    '.html': 'text/html; charset=utf-8',
+    '.js': 'text/javascript; charset=utf-8',
+    '.map': 'application/json',
+    '.wasm': 'application/wasm',
+};
+
+createServer((request, response) => {
+    response.setHeader('Cross-Origin-Opener-Policy', 'same-origin');
+    response.setHeader('Cross-Origin-Embedder-Policy', 'require-corp');
+
+    const pathname = new URL(request.url ?? '/', 'http://localhost').pathname;
+    const relativePath = pathname === '/' ? 'examples/browser.html' : pathname.slice(1);
+    const file = resolve(packageRoot, relativePath);
+    if (!file.startsWith(`${packageRoot}${sep}`) || !existsSync(file) || !statSync(file).isFile()) {
+        response.writeHead(404).end('Not found');
+        return;
+    }
+
+    const compressedWasm = file.endsWith('.wasm.br');
+    response.writeHead(200, {
+        ...(compressedWasm ? { 'Content-Encoding': 'br' } : {}),
+        'Content-Length': statSync(file).size,
+        'Content-Type': compressedWasm ? 'application/wasm' : contentTypes[extname(file)] ?? 'application/octet-stream',
+    });
+    createReadStream(file).pipe(response);
+}).listen(port, '127.0.0.1', () => {
+    console.log(`HyperDB browser example: http://127.0.0.1:${port}/`);
+});

package/package.json

@@ -0,0 +1,41 @@
+{
+    "name": "hyperdb-wasm",
+    "version": "0.0.26268-dev.remotes-origin-wasm.gcf8d815a",
+    "type": "module",
+    "engines": {
+        "node": ">=24"
+    },
+    "files": [
+        "dist",
+        "examples"
+    ],
+    "exports": {
+        ".": {
+            "browser": {
+                "types": "./dist/index.d.ts",
+                "default": "./dist/browser.js"
+            },
+            "node": {
+                "types": "./dist/index.node.d.ts",
+                "default": "./dist/node.js"
+            },
+            "types": "./dist/index.d.ts",
+            "default": "./dist/browser.js"
+        },
+        "./raw": {
+            "browser": {
+                "types": "./dist/raw.browser.d.ts",
+                "default": "./dist/raw.browser.js"
+            },
+            "node": {
+                "types": "./dist/raw.node.d.ts",
+                "default": "./dist/raw.node.js"
+            },
+            "types": "./dist/raw.browser.d.ts",
+            "default": "./dist/raw.browser.js"
+        }
+    },
+    "dependencies": {
+        "apache-arrow": "^19.0.1"
+    }
+}