hyperdb-wasm 0.0.26282-dev.wasm.g959538a5 → 0.0.26297-dev.wasm.g08f2873d

package/dist/api.d.ts

@@ -1,5 +1,5 @@
-import { Table } from 'apache-arrow';
-import type { AsyncRecordBatchStreamReader, DataType, TypeMap } from 'apache-arrow';
+import { DataType, Table } from 'apache-arrow';
+import type { AsyncRecordBatchStreamReader, TypeMap } from 'apache-arrow';
 import type { HyperDBEngineClient } from './client.js';
 import type { BrowserClientOptions } from './browser.js';
 import type { WasmResult } from './protocol.js';
@@ -7,23 +7,47 @@ export interface HyperDBClient {
     ready(): Promise<void>;
     connect(): Promise<WasmResult>;
     disconnect(connection: number): Promise<WasmResult>;
+    createDatabase(databaseName: string, persistent: boolean): Promise<WasmResult>;
+    openDatabase(databaseName: string): Promise<WasmResult>;
+    checkpointDatabase(databaseName: string): Promise<WasmResult>;
+    dropDatabase(databaseName: string): Promise<WasmResult>;
+    attachDatabase(connection: number, databaseName: string, alias: string): Promise<WasmResult>;
+    detachDatabase(connection: number, alias: string): Promise<WasmResult>;
     startQuery(connection: number, sql: string): Promise<WasmResult>;
+    insertArrowIPCFromPath(connection: number, path: string, name: string, schema: string | null, create: boolean, internal?: boolean): Promise<WasmResult>;
+    createTemporaryFile(bytes: Uint8Array): Promise<WasmResult>;
+    removeFile(path: string): Promise<WasmResult>;
     pollQuery(query: number): Promise<WasmResult>;
     cancelQuery(query: number): Promise<WasmResult>;
     releaseQuery(query: number): Promise<WasmResult>;
+    shutdown(): Promise<WasmResult>;
     terminate(): Promise<void>;
 }
+export interface ArrowInsertOptions {
+    readonly name: string;
+    readonly schema?: string;
+    readonly create?: boolean;
+}
+export interface DatabaseCreateOptions {
+    readonly storage?: 'memory' | 'persistent';
+}
 export declare class HyperDB {
     private readonly client;
     private terminated;
     private termination;
+    private activeDatabaseOperations;
     private readonly connections;
     constructor(client: HyperDBClient);
     static create(options?: BrowserClientOptions): Promise<HyperDB>;
     ready(): Promise<void>;
     connect(): Promise<HyperDBConnection>;
+    createDatabase(databaseName: string, options?: DatabaseCreateOptions): Promise<void>;
+    openDatabase(databaseName: string): Promise<void>;
+    checkpointDatabase(databaseName: string): Promise<void>;
+    dropDatabase(databaseName: string): Promise<void>;
     terminate(): Promise<void>;
     private ensureRunning;
+    private runDatabaseOperation;
 }
 export declare class HyperDBConnection {
     private readonly client;
@@ -34,6 +58,8 @@ export declare class HyperDBConnection {
     private closing;
     constructor(client: HyperDBClient, connection: number, onClose?: () => void);
     close(): Promise<void>;
+    attachDatabase(databaseName: string, alias?: string): Promise<void>;
+    detachDatabase(alias: string): Promise<void>;
     sendIPC(sql: string): AsyncIterable<Uint8Array>;
     queryIPC(sql: string): Promise<Uint8Array>;
     send<T extends TypeMap = {
@@ -42,6 +68,9 @@ export declare class HyperDBConnection {
     query<T extends TypeMap = {
         [key: string]: DataType;
     }>(sql: string): Promise<Table<T>>;
+    insertArrowTable(table: Table, options: ArrowInsertOptions): Promise<void>;
+    insertArrowIPCFromPath(path: string, options: ArrowInsertOptions): Promise<void>;
+    private insertArrowIPCFromPathInternal;
     private ensureAvailable;
     private release;
     closeForTermination(): Promise<void>;

package/dist/browser.js

@@ -95,9 +95,40 @@ var init_client = __esm({
       disconnect(connection) {
         return this.request({ id: this.nextId++, type: "disconnect", connection });
       }
+      createDatabase(databaseName, persistent) {
+        return this.request({ id: this.nextId++, type: "createDatabase", databaseName, persistent });
+      }
+      openDatabase(databaseName) {
+        return this.request({ id: this.nextId++, type: "openDatabase", databaseName });
+      }
+      checkpointDatabase(databaseName) {
+        return this.request({ id: this.nextId++, type: "checkpointDatabase", databaseName });
+      }
+      dropDatabase(databaseName) {
+        return this.request({ id: this.nextId++, type: "dropDatabase", databaseName });
+      }
+      attachDatabase(connection, databaseName, alias) {
+        return this.request({ id: this.nextId++, type: "attachDatabase", connection, databaseName, alias });
+      }
+      detachDatabase(connection, alias) {
+        return this.request({ id: this.nextId++, type: "detachDatabase", connection, alias });
+      }
+      shutdown() {
+        return this.request({ id: this.nextId++, type: "shutdown" });
+      }
       startQuery(connection, sql) {
         return this.request({ id: this.nextId++, type: "startQuery", connection, sql });
       }
+      insertArrowIPCFromPath(connection, path, name, schema, create, internal = false) {
+        return this.request({ id: this.nextId++, type: "insertArrowIPCFromPath", connection, path, name, schema, create, internal });
+      }
+      createTemporaryFile(bytes) {
+        const payload = bytes.slice();
+        return this.request({ id: this.nextId++, type: "createTemporaryFile", bytes: payload }, [payload.buffer]);
+      }
+      removeFile(path) {
+        return this.request({ id: this.nextId++, type: "removeFile", path });
+      }
       pollQuery(query) {
         return this.request({ id: this.nextId++, type: "pollQuery", query });
       }
@@ -111,12 +142,12 @@ var init_client = __esm({
         this.fail(new Error("engine worker terminated"));
         await this.transport.terminate();
       }
-      request(request) {
+      request(request, transfer = []) {
         if (this.failed) return Promise.reject(this.failed);
         return new Promise((resolve, reject) => {
           this.pending.set(request.id, { resolve, reject });
           try {
-            this.transport.postMessage(request);
+            this.transport.postMessage(request, transfer);
           } catch (error) {
             this.pending.delete(request.id);
             reject(error instanceof Error ? error : new Error(String(error)));
@@ -3858,6 +3889,11 @@ Timestamp_[_o] = ((proto) => {
   proto.ArrayType = BigInt64Array;
   return proto[Symbol.toStringTag] = "Timestamp";
 })(Timestamp_.prototype);
+var TimestampMillisecond = class extends Timestamp_ {
+  constructor(timezone) {
+    super(TimeUnit.MILLISECOND, timezone);
+  }
+};
 var Interval_ = class extends DataType {
   constructor(unit) {
     super(Type2.Interval);
@@ -6397,6 +6433,63 @@ var MemoizedVector = class _MemoizedVector extends Vector {
     });
   }
 };
+function makeVector(init) {
+  if (init) {
+    if (init instanceof Data) {
+      return new Vector([init]);
+    }
+    if (init instanceof Vector) {
+      return new Vector(init.data);
+    }
+    if (init.type instanceof DataType) {
+      return new Vector([makeData(init)]);
+    }
+    if (Array.isArray(init)) {
+      return new Vector(init.flatMap((v) => unwrapInputs(v)));
+    }
+    if (ArrayBuffer.isView(init)) {
+      if (init instanceof DataView) {
+        init = new Uint8Array(init.buffer);
+      }
+      const props = { offset: 0, length: init.length, nullCount: -1, data: init };
+      if (init instanceof Int8Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int8() }))]);
+      }
+      if (init instanceof Int16Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int16() }))]);
+      }
+      if (init instanceof Int32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int32() }))]);
+      }
+      if (init instanceof BigInt64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int64() }))]);
+      }
+      if (init instanceof Uint8Array || init instanceof Uint8ClampedArray) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint8() }))]);
+      }
+      if (init instanceof Uint16Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint16() }))]);
+      }
+      if (init instanceof Uint32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint32() }))]);
+      }
+      if (init instanceof BigUint64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint64() }))]);
+      }
+      if (init instanceof Float32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Float32() }))]);
+      }
+      if (init instanceof Float64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Float64() }))]);
+      }
+      throw new Error("Unrecognized input");
+    }
+  }
+  throw new Error("Unrecognized input");
+}
+function unwrapInputs(x) {
+  return x instanceof Data ? [x] : x instanceof Vector ? x.data : makeVector(x).data;
+}
 
 // ../node_modules/apache-arrow/builder/valid.mjs
 function createIsValidFunction(nullValues) {
@@ -8872,6 +8965,106 @@ function makeBuilder(options) {
   }
   return builder;
 }
+function vectorFromArray(init, type) {
+  if (init instanceof Data || init instanceof Vector || init.type instanceof DataType || ArrayBuffer.isView(init)) {
+    return makeVector(init);
+  }
+  const options = { type: type !== null && type !== void 0 ? type : inferType(init), nullValues: [null] };
+  const chunks = [...builderThroughIterable(options)(init)];
+  const vector = chunks.length === 1 ? chunks[0] : chunks.reduce((a, b) => a.concat(b));
+  if (DataType.isDictionary(vector.type)) {
+    return vector.memoize();
+  }
+  return vector;
+}
+function inferType(value) {
+  if (value.length === 0) {
+    return new Null2();
+  }
+  let nullsCount = 0;
+  let arraysCount = 0;
+  let objectsCount = 0;
+  let numbersCount = 0;
+  let stringsCount = 0;
+  let bigintsCount = 0;
+  let booleansCount = 0;
+  let datesCount = 0;
+  for (const val of value) {
+    if (val == null) {
+      ++nullsCount;
+      continue;
+    }
+    switch (typeof val) {
+      case "bigint":
+        ++bigintsCount;
+        continue;
+      case "boolean":
+        ++booleansCount;
+        continue;
+      case "number":
+        ++numbersCount;
+        continue;
+      case "string":
+        ++stringsCount;
+        continue;
+      case "object":
+        if (Array.isArray(val)) {
+          ++arraysCount;
+        } else if (Object.prototype.toString.call(val) === "[object Date]") {
+          ++datesCount;
+        } else {
+          ++objectsCount;
+        }
+        continue;
+    }
+    throw new TypeError("Unable to infer Vector type from input values, explicit type declaration expected.");
+  }
+  if (numbersCount + nullsCount === value.length) {
+    return new Float64();
+  } else if (stringsCount + nullsCount === value.length) {
+    return new Dictionary(new Utf82(), new Int32());
+  } else if (bigintsCount + nullsCount === value.length) {
+    return new Int64();
+  } else if (booleansCount + nullsCount === value.length) {
+    return new Bool2();
+  } else if (datesCount + nullsCount === value.length) {
+    return new TimestampMillisecond();
+  } else if (arraysCount + nullsCount === value.length) {
+    const array = value;
+    const childType = inferType(array[array.findIndex((ary) => ary != null)]);
+    if (array.every((ary) => ary == null || compareTypes(childType, inferType(ary)))) {
+      return new List2(new Field2("", childType, true));
+    }
+  } else if (objectsCount + nullsCount === value.length) {
+    const fields = /* @__PURE__ */ new Map();
+    for (const row of value) {
+      for (const key of Object.keys(row)) {
+        if (!fields.has(key) && row[key] != null) {
+          fields.set(key, new Field2(key, inferType([row[key]]), true));
+        }
+      }
+    }
+    return new Struct([...fields.values()]);
+  }
+  throw new TypeError("Unable to infer Vector type from input values, explicit type declaration expected.");
+}
+function builderThroughIterable(options) {
+  const { ["queueingStrategy"]: queueingStrategy = "count" } = options;
+  const { ["highWaterMark"]: highWaterMark = queueingStrategy !== "bytes" ? Number.POSITIVE_INFINITY : Math.pow(2, 14) } = options;
+  const sizeProperty = queueingStrategy !== "bytes" ? "length" : "byteLength";
+  return function* (source) {
+    let numChunks = 0;
+    const builder = makeBuilder(options);
+    for (const value of source) {
+      if (builder.append(value)[sizeProperty] >= highWaterMark) {
+        ++numChunks && (yield builder.toVector());
+      }
+    }
+    if (builder.finish().length > 0 || numChunks === 0) {
+      yield builder.toVector();
+    }
+  };
+}
 
 // ../node_modules/apache-arrow/util/recordbatch.mjs
 function distributeVectorsIntoRecordBatches(schema, vecs) {
@@ -11858,6 +12051,11 @@ function recordBatchWriterThroughDOMStream(writableStrategy, readableStrategy) {
   }
 }
 
+// ../node_modules/apache-arrow/ipc/serialization.mjs
+function tableToIPC(table, type = "stream") {
+  return (type === "stream" ? RecordBatchStreamWriter : RecordBatchFileWriter).writeAll(table).toUint8Array(true);
+}
+
 // ../node_modules/apache-arrow/Arrow.mjs
 var util = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, bn_exports), int_exports), bit_exports), math_exports), buffer_exports), vector_exports), pretty_exports), {
   compareSchemas,
@@ -11876,6 +12074,36 @@ RecordBatchFileWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 RecordBatchStreamWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 
 // api.ts
+function normalizeArrowInsertOptions(options) {
+  if (!options.name) throw new Error("Arrow insert table name must not be empty");
+  return { name: options.name, schema: options.schema ?? null, create: options.create ?? true };
+}
+function validateDatabaseName(databaseName) {
+  if (!databaseName) throw new Error("Database name must not be empty");
+  if (databaseName.includes("\0")) throw new Error("Database name must not contain NUL");
+}
+function validateDatabaseAlias(alias) {
+  if (!alias) throw new Error("Database alias must not be empty");
+  if (alias.includes("\0")) throw new Error("Database alias must not contain NUL");
+}
+function materializeDictionaryColumns(table) {
+  const dictionaryColumns = table.schema.fields.map((field, index) => DataType.isDictionary(field.type) ? index : -1).filter((index) => index !== -1);
+  if (dictionaryColumns.length === 0) return table;
+  const fields = table.schema.fields.map((field) => DataType.isDictionary(field.type) ? field.clone({ type: field.type.dictionary }) : field);
+  const batches = table.batches.map((batch) => {
+    let result = batch;
+    for (const index of dictionaryColumns) {
+      const column = result.getChildAt(index);
+      const sourceField = table.schema.fields[index];
+      if (!column || !sourceField) throw new Error(`Arrow table is missing column ${index}`);
+      const type = sourceField.type;
+      const materialized = vectorFromArray([...column], type.dictionary);
+      result = result.setChildAt(index, materialized);
+    }
+    return result;
+  });
+  return new Table(new Schema2(fields, table.schema.metadata, null, table.schema.metadataVersion), batches);
+}
 function readHandleFromResult(result, operation) {
   if (result.state !== "ok") throw readErrorFromResult(result, operation);
   if (result.payload.byteLength !== 4) throw new Error(`${operation} returned an invalid handle`);
@@ -11898,6 +12126,7 @@ var HyperDB = class _HyperDB {
   client;
   terminated = false;
   termination;
+  activeDatabaseOperations = 0;
   connections = /* @__PURE__ */ new Set();
   static async create(options = {}) {
     const { createBrowserClient: createBrowserClient2 } = await Promise.resolve().then(() => (init_browser(), browser_exports));
@@ -11916,8 +12145,36 @@ var HyperDB = class _HyperDB {
     this.connections.add(result);
     return result;
   }
+  async createDatabase(databaseName, options = {}) {
+    validateDatabaseName(databaseName);
+    if (options.storage !== void 0 && options.storage !== "memory" && options.storage !== "persistent") {
+      throw new Error(`Unsupported database storage: ${String(options.storage)}`);
+    }
+    await this.runDatabaseOperation(async () => {
+      expectOk(await this.client.createDatabase(databaseName, options.storage === "persistent"), "create database");
+    });
+  }
+  async openDatabase(databaseName) {
+    validateDatabaseName(databaseName);
+    await this.runDatabaseOperation(async () => {
+      expectOk(await this.client.openDatabase(databaseName), "open database");
+    });
+  }
+  async checkpointDatabase(databaseName) {
+    validateDatabaseName(databaseName);
+    await this.runDatabaseOperation(async () => {
+      expectOk(await this.client.checkpointDatabase(databaseName), "checkpoint database");
+    });
+  }
+  async dropDatabase(databaseName) {
+    validateDatabaseName(databaseName);
+    await this.runDatabaseOperation(async () => {
+      expectOk(await this.client.dropDatabase(databaseName), "drop database");
+    });
+  }
   async terminate() {
     if (this.terminated) return;
+    if (this.activeDatabaseOperations !== 0) throw new Error("database has an active operation");
     if (!this.termination) {
       this.termination = (async () => {
         try {
@@ -11926,6 +12183,12 @@ var HyperDB = class _HyperDB {
           this.termination = void 0;
           throw error;
         }
+        try {
+          expectOk(await this.client.shutdown(), "shutdown");
+        } catch (error) {
+          this.termination = void 0;
+          throw error;
+        }
         this.terminated = true;
         await this.client.terminate();
       })();
@@ -11934,6 +12197,16 @@ var HyperDB = class _HyperDB {
   }
   ensureRunning() {
     if (this.terminated) throw new Error("database is terminated");
+    if (this.termination) throw new Error("database is terminating");
+  }
+  async runDatabaseOperation(operation) {
+    this.ensureRunning();
+    ++this.activeDatabaseOperations;
+    try {
+      await operation();
+    } finally {
+      --this.activeDatabaseOperations;
+    }
   }
 };
 var HyperDBConnection = class {
@@ -11950,7 +12223,7 @@ var HyperDBConnection = class {
   closing;
   async close() {
     if (this.closed) return;
-    if (this.active) throw new Error("connection has an active query");
+    if (this.active) throw new Error("connection has an active operation");
     if (!this.closing) {
       this.closing = (async () => {
         try {
@@ -11965,6 +12238,25 @@ var HyperDBConnection = class {
     }
     await this.closing;
   }
+  async attachDatabase(databaseName, alias = databaseName) {
+    validateDatabaseName(databaseName);
+    validateDatabaseAlias(alias);
+    this.ensureAvailable();
+    try {
+      expectOk(await this.client.attachDatabase(this.connection, databaseName, alias), "attach database");
+    } finally {
+      this.active = false;
+    }
+  }
+  async detachDatabase(alias) {
+    validateDatabaseAlias(alias);
+    this.ensureAvailable();
+    try {
+      expectOk(await this.client.detachDatabase(this.connection, alias), "detach database");
+    } finally {
+      this.active = false;
+    }
+  }
   sendIPC(sql) {
     const connection = this;
     return {
@@ -12051,11 +12343,55 @@ var HyperDBConnection = class {
     for await (const batch of reader) batches.push(batch);
     return new Table(reader.schema, batches);
   }
+  async insertArrowTable(table, options) {
+    this.ensureAvailable();
+    let path;
+    try {
+      const result = await this.client.createTemporaryFile(tableToIPC(materializeDictionaryColumns(table), "stream"));
+      expectOk(result, "create temporary Arrow IPC file");
+      path = new TextDecoder().decode(result.payload);
+      await this.insertArrowIPCFromPathInternal(path, options, true, false);
+    } finally {
+      try {
+        if (path !== void 0) expectOk(await this.client.removeFile(path), "remove temporary Arrow IPC file");
+      } finally {
+        this.active = false;
+      }
+    }
+  }
+  async insertArrowIPCFromPath(path, options) {
+    await this.insertArrowIPCFromPathInternal(path, options, false);
+  }
+  async insertArrowIPCFromPathInternal(path, options, internal, acquire = true) {
+    if (!path) throw new Error("Arrow IPC input path must not be empty");
+    const normalized = normalizeArrowInsertOptions(options);
+    if (acquire) this.ensureAvailable();
+    let query;
+    try {
+      query = readHandleFromResult(await this.client.insertArrowIPCFromPath(this.connection, path, normalized.name, normalized.schema, normalized.create, internal), "insert Arrow IPC from path");
+      for (; ; ) {
+        let result = await this.client.pollQuery(query);
+        while (result.state === "pending") {
+          await yieldToEventLoop();
+          result = await this.client.pollQuery(query);
+        }
+        if (result.state === "done") return;
+        if (result.state === "chunk") continue;
+        throw readErrorFromResult(result, "insert Arrow IPC from path");
+      }
+    } finally {
+      try {
+        if (query !== void 0) await this.release(query);
+      } finally {
+        if (acquire) this.active = false;
+      }
+    }
+  }
   ensureAvailable() {
     if (this.closed) throw new Error("connection is closed");
     if (this.closing) throw new Error("connection is closing");
     if (this.active) {
-      throw new Error("connection has an active query");
+      throw new Error("connection has an active operation");
     }
     this.active = true;
   }
@@ -12070,7 +12406,7 @@ var HyperDBConnection = class {
     }
   }
   async closeForTermination() {
-    if (this.active) throw new Error("cannot terminate while a connection has an active query");
+    if (this.active) throw new Error("cannot terminate while a connection has an active operation");
     await this.close();
   }
 };