hyperdb-wasm 0.0.26281-dev.wasm.g39d82636 → 0.0.26284-dev.wasm.gf1f9d795

package/dist/api.d.ts

@@ -1,5 +1,5 @@
-import { Table } from 'apache-arrow';
-import type { AsyncRecordBatchStreamReader, DataType, TypeMap } from 'apache-arrow';
+import { DataType, Table } from 'apache-arrow';
+import type { AsyncRecordBatchStreamReader, TypeMap } from 'apache-arrow';
 import type { HyperDBEngineClient } from './client.js';
 import type { BrowserClientOptions } from './browser.js';
 import type { WasmResult } from './protocol.js';
@@ -8,11 +8,18 @@ export interface HyperDBClient {
     connect(): Promise<WasmResult>;
     disconnect(connection: number): Promise<WasmResult>;
     startQuery(connection: number, sql: string): Promise<WasmResult>;
+    appendArrowIPC(connection: number, chunk: Uint8Array): Promise<WasmResult>;
+    finishArrowIPC(connection: number, name: string, schema: string | null, create: boolean): Promise<WasmResult>;
     pollQuery(query: number): Promise<WasmResult>;
     cancelQuery(query: number): Promise<WasmResult>;
     releaseQuery(query: number): Promise<WasmResult>;
     terminate(): Promise<void>;
 }
+export interface ArrowInsertOptions {
+    readonly name: string;
+    readonly schema?: string;
+    readonly create?: boolean;
+}
 export declare class HyperDB {
     private readonly client;
     private terminated;
@@ -31,6 +38,8 @@ export declare class HyperDBConnection {
     private readonly onClose;
     private closed;
     private active;
+    private arrowInsertOptions;
+    private appendingArrow;
     private closing;
     constructor(client: HyperDBClient, connection: number, onClose?: () => void);
     close(): Promise<void>;
@@ -42,6 +51,9 @@ export declare class HyperDBConnection {
     query<T extends TypeMap = {
         [key: string]: DataType;
     }>(sql: string): Promise<Table<T>>;
+    insertArrowTable(table: Table, options: ArrowInsertOptions): Promise<void>;
+    insertArrowFromIPCStream(chunk: Uint8Array, options: ArrowInsertOptions): Promise<void>;
+    finishArrowIPCStream(options?: ArrowInsertOptions): Promise<void>;
     private ensureAvailable;
     private release;
     closeForTermination(): Promise<void>;

package/dist/browser.js

@@ -98,6 +98,13 @@ var init_client = __esm({
       startQuery(connection, sql) {
         return this.request({ id: this.nextId++, type: "startQuery", connection, sql });
       }
+      appendArrowIPC(connection, chunk) {
+        const payload = chunk.slice();
+        return this.request({ id: this.nextId++, type: "appendArrowIPC", connection, chunk: payload }, [payload.buffer]);
+      }
+      finishArrowIPC(connection, name, schema, create) {
+        return this.request({ id: this.nextId++, type: "finishArrowIPC", connection, name, schema, create });
+      }
       pollQuery(query) {
         return this.request({ id: this.nextId++, type: "pollQuery", query });
       }
@@ -111,12 +118,12 @@ var init_client = __esm({
         this.fail(new Error("engine worker terminated"));
         await this.transport.terminate();
       }
-      request(request) {
+      request(request, transfer = []) {
         if (this.failed) return Promise.reject(this.failed);
         return new Promise((resolve, reject) => {
           this.pending.set(request.id, { resolve, reject });
           try {
-            this.transport.postMessage(request);
+            this.transport.postMessage(request, transfer);
           } catch (error) {
             this.pending.delete(request.id);
             reject(error instanceof Error ? error : new Error(String(error)));
@@ -3858,6 +3865,11 @@ Timestamp_[_o] = ((proto) => {
   proto.ArrayType = BigInt64Array;
   return proto[Symbol.toStringTag] = "Timestamp";
 })(Timestamp_.prototype);
+var TimestampMillisecond = class extends Timestamp_ {
+  constructor(timezone) {
+    super(TimeUnit.MILLISECOND, timezone);
+  }
+};
 var Interval_ = class extends DataType {
   constructor(unit) {
     super(Type2.Interval);
@@ -6397,6 +6409,63 @@ var MemoizedVector = class _MemoizedVector extends Vector {
     });
   }
 };
+function makeVector(init) {
+  if (init) {
+    if (init instanceof Data) {
+      return new Vector([init]);
+    }
+    if (init instanceof Vector) {
+      return new Vector(init.data);
+    }
+    if (init.type instanceof DataType) {
+      return new Vector([makeData(init)]);
+    }
+    if (Array.isArray(init)) {
+      return new Vector(init.flatMap((v) => unwrapInputs(v)));
+    }
+    if (ArrayBuffer.isView(init)) {
+      if (init instanceof DataView) {
+        init = new Uint8Array(init.buffer);
+      }
+      const props = { offset: 0, length: init.length, nullCount: -1, data: init };
+      if (init instanceof Int8Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int8() }))]);
+      }
+      if (init instanceof Int16Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int16() }))]);
+      }
+      if (init instanceof Int32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int32() }))]);
+      }
+      if (init instanceof BigInt64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Int64() }))]);
+      }
+      if (init instanceof Uint8Array || init instanceof Uint8ClampedArray) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint8() }))]);
+      }
+      if (init instanceof Uint16Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint16() }))]);
+      }
+      if (init instanceof Uint32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint32() }))]);
+      }
+      if (init instanceof BigUint64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Uint64() }))]);
+      }
+      if (init instanceof Float32Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Float32() }))]);
+      }
+      if (init instanceof Float64Array) {
+        return new Vector([makeData(Object.assign(Object.assign({}, props), { type: new Float64() }))]);
+      }
+      throw new Error("Unrecognized input");
+    }
+  }
+  throw new Error("Unrecognized input");
+}
+function unwrapInputs(x) {
+  return x instanceof Data ? [x] : x instanceof Vector ? x.data : makeVector(x).data;
+}
 
 // ../node_modules/apache-arrow/builder/valid.mjs
 function createIsValidFunction(nullValues) {
@@ -8872,6 +8941,106 @@ function makeBuilder(options) {
   }
   return builder;
 }
+function vectorFromArray(init, type) {
+  if (init instanceof Data || init instanceof Vector || init.type instanceof DataType || ArrayBuffer.isView(init)) {
+    return makeVector(init);
+  }
+  const options = { type: type !== null && type !== void 0 ? type : inferType(init), nullValues: [null] };
+  const chunks = [...builderThroughIterable(options)(init)];
+  const vector = chunks.length === 1 ? chunks[0] : chunks.reduce((a, b) => a.concat(b));
+  if (DataType.isDictionary(vector.type)) {
+    return vector.memoize();
+  }
+  return vector;
+}
+function inferType(value) {
+  if (value.length === 0) {
+    return new Null2();
+  }
+  let nullsCount = 0;
+  let arraysCount = 0;
+  let objectsCount = 0;
+  let numbersCount = 0;
+  let stringsCount = 0;
+  let bigintsCount = 0;
+  let booleansCount = 0;
+  let datesCount = 0;
+  for (const val of value) {
+    if (val == null) {
+      ++nullsCount;
+      continue;
+    }
+    switch (typeof val) {
+      case "bigint":
+        ++bigintsCount;
+        continue;
+      case "boolean":
+        ++booleansCount;
+        continue;
+      case "number":
+        ++numbersCount;
+        continue;
+      case "string":
+        ++stringsCount;
+        continue;
+      case "object":
+        if (Array.isArray(val)) {
+          ++arraysCount;
+        } else if (Object.prototype.toString.call(val) === "[object Date]") {
+          ++datesCount;
+        } else {
+          ++objectsCount;
+        }
+        continue;
+    }
+    throw new TypeError("Unable to infer Vector type from input values, explicit type declaration expected.");
+  }
+  if (numbersCount + nullsCount === value.length) {
+    return new Float64();
+  } else if (stringsCount + nullsCount === value.length) {
+    return new Dictionary(new Utf82(), new Int32());
+  } else if (bigintsCount + nullsCount === value.length) {
+    return new Int64();
+  } else if (booleansCount + nullsCount === value.length) {
+    return new Bool2();
+  } else if (datesCount + nullsCount === value.length) {
+    return new TimestampMillisecond();
+  } else if (arraysCount + nullsCount === value.length) {
+    const array = value;
+    const childType = inferType(array[array.findIndex((ary) => ary != null)]);
+    if (array.every((ary) => ary == null || compareTypes(childType, inferType(ary)))) {
+      return new List2(new Field2("", childType, true));
+    }
+  } else if (objectsCount + nullsCount === value.length) {
+    const fields = /* @__PURE__ */ new Map();
+    for (const row of value) {
+      for (const key of Object.keys(row)) {
+        if (!fields.has(key) && row[key] != null) {
+          fields.set(key, new Field2(key, inferType([row[key]]), true));
+        }
+      }
+    }
+    return new Struct([...fields.values()]);
+  }
+  throw new TypeError("Unable to infer Vector type from input values, explicit type declaration expected.");
+}
+function builderThroughIterable(options) {
+  const { ["queueingStrategy"]: queueingStrategy = "count" } = options;
+  const { ["highWaterMark"]: highWaterMark = queueingStrategy !== "bytes" ? Number.POSITIVE_INFINITY : Math.pow(2, 14) } = options;
+  const sizeProperty = queueingStrategy !== "bytes" ? "length" : "byteLength";
+  return function* (source) {
+    let numChunks = 0;
+    const builder = makeBuilder(options);
+    for (const value of source) {
+      if (builder.append(value)[sizeProperty] >= highWaterMark) {
+        ++numChunks && (yield builder.toVector());
+      }
+    }
+    if (builder.finish().length > 0 || numChunks === 0) {
+      yield builder.toVector();
+    }
+  };
+}
 
 // ../node_modules/apache-arrow/util/recordbatch.mjs
 function distributeVectorsIntoRecordBatches(schema, vecs) {
@@ -11858,6 +12027,11 @@ function recordBatchWriterThroughDOMStream(writableStrategy, readableStrategy) {
   }
 }
 
+// ../node_modules/apache-arrow/ipc/serialization.mjs
+function tableToIPC(table, type = "stream") {
+  return (type === "stream" ? RecordBatchStreamWriter : RecordBatchFileWriter).writeAll(table).toUint8Array(true);
+}
+
 // ../node_modules/apache-arrow/Arrow.mjs
 var util = Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign(Object.assign({}, bn_exports), int_exports), bit_exports), math_exports), buffer_exports), vector_exports), pretty_exports), {
   compareSchemas,
@@ -11876,6 +12050,31 @@ RecordBatchFileWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 RecordBatchStreamWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 
 // api.ts
+function normalizeArrowInsertOptions(options) {
+  if (!options.name) throw new Error("Arrow insert table name must not be empty");
+  return { name: options.name, schema: options.schema ?? null, create: options.create ?? true };
+}
+function equalArrowInsertOptions(left, right) {
+  return left.name === right.name && left.schema === right.schema && left.create === right.create;
+}
+function materializeDictionaryColumns(table) {
+  const dictionaryColumns = table.schema.fields.map((field, index) => DataType.isDictionary(field.type) ? index : -1).filter((index) => index !== -1);
+  if (dictionaryColumns.length === 0) return table;
+  const fields = table.schema.fields.map((field) => DataType.isDictionary(field.type) ? field.clone({ type: field.type.dictionary }) : field);
+  const batches = table.batches.map((batch) => {
+    let result = batch;
+    for (const index of dictionaryColumns) {
+      const column = result.getChildAt(index);
+      const sourceField = table.schema.fields[index];
+      if (!column || !sourceField) throw new Error(`Arrow table is missing column ${index}`);
+      const type = sourceField.type;
+      const materialized = vectorFromArray([...column], type.dictionary);
+      result = result.setChildAt(index, materialized);
+    }
+    return result;
+  });
+  return new Table(new Schema2(fields, table.schema.metadata, null, table.schema.metadataVersion), batches);
+}
 function readHandleFromResult(result, operation) {
   if (result.state !== "ok") throw readErrorFromResult(result, operation);
   if (result.payload.byteLength !== 4) throw new Error(`${operation} returned an invalid handle`);
@@ -11947,10 +12146,13 @@ var HyperDBConnection = class {
   onClose;
   closed = false;
   active = false;
+  arrowInsertOptions;
+  appendingArrow = false;
   closing;
   async close() {
     if (this.closed) return;
     if (this.active) throw new Error("connection has an active query");
+    if (this.arrowInsertOptions) throw new Error("connection has an unfinished Arrow IPC stream");
     if (!this.closing) {
       this.closing = (async () => {
         try {
@@ -12051,6 +12253,60 @@ var HyperDBConnection = class {
     for await (const batch of reader) batches.push(batch);
     return new Table(reader.schema, batches);
   }
+  async insertArrowTable(table, options) {
+    await this.insertArrowFromIPCStream(tableToIPC(materializeDictionaryColumns(table), "stream"), options);
+    await this.finishArrowIPCStream();
+  }
+  async insertArrowFromIPCStream(chunk, options) {
+    if (this.closed) throw new Error("connection is closed");
+    if (this.closing) throw new Error("connection is closing");
+    if (this.active) throw new Error("connection has an active query");
+    if (this.appendingArrow) throw new Error("connection is appending Arrow IPC");
+    const normalized = normalizeArrowInsertOptions(options);
+    if (this.arrowInsertOptions && !equalArrowInsertOptions(this.arrowInsertOptions, normalized)) {
+      throw new Error("Arrow insert options must match the pending stream");
+    }
+    this.appendingArrow = true;
+    try {
+      expectOk(await this.client.appendArrowIPC(this.connection, chunk), "append Arrow IPC");
+      this.arrowInsertOptions = normalized;
+    } finally {
+      this.appendingArrow = false;
+    }
+  }
+  async finishArrowIPCStream(options) {
+    if (this.closed) throw new Error("connection is closed");
+    if (this.closing) throw new Error("connection is closing");
+    if (this.active) throw new Error("connection has an active query");
+    if (this.appendingArrow) throw new Error("connection is appending Arrow IPC");
+    const normalized = options ? normalizeArrowInsertOptions(options) : this.arrowInsertOptions;
+    if (!normalized) throw new Error("connection has no Arrow IPC input");
+    if (this.arrowInsertOptions && !equalArrowInsertOptions(this.arrowInsertOptions, normalized)) {
+      throw new Error("Arrow insert options must match the pending stream");
+    }
+    this.active = true;
+    let query;
+    try {
+      query = readHandleFromResult(await this.client.finishArrowIPC(this.connection, normalized.name, normalized.schema, normalized.create), "finish Arrow IPC");
+      this.arrowInsertOptions = void 0;
+      for (; ; ) {
+        let result = await this.client.pollQuery(query);
+        while (result.state === "pending") {
+          await yieldToEventLoop();
+          result = await this.client.pollQuery(query);
+        }
+        if (result.state === "done") return;
+        if (result.state === "chunk") continue;
+        throw readErrorFromResult(result, "finish Arrow IPC");
+      }
+    } finally {
+      try {
+        if (query !== void 0) await this.release(query);
+      } finally {
+        this.active = false;
+      }
+    }
+  }
   ensureAvailable() {
     if (this.closed) throw new Error("connection is closed");
     if (this.closing) throw new Error("connection is closing");
@@ -12071,6 +12327,7 @@ var HyperDBConnection = class {
   }
   async closeForTermination() {
     if (this.active) throw new Error("cannot terminate while a connection has an active query");
+    if (this.arrowInsertOptions) throw new Error("cannot terminate while a connection has an unfinished Arrow IPC stream");
     await this.close();
   }
 };

package/dist/browser_worker.js

@@ -12,6 +12,8 @@
     connect;
     disconnect;
     startQuery;
+    appendArrowIPC;
+    finishArrowIPC;
     pollQuery;
     cancelQuery;
     releaseQuery;
@@ -23,6 +25,8 @@
       this.connect = module.cwrap("hyperdb_wasm_connect", "pointer", []);
       this.disconnect = module.cwrap("hyperdb_wasm_disconnect", "pointer", ["number"]);
       this.startQuery = module.cwrap("hyperdb_wasm_query_start", "pointer", ["number", "string"]);
+      this.appendArrowIPC = module.cwrap("hyperdb_wasm_arrow_insert_append", "pointer", ["number", "number", "number"]);
+      this.finishArrowIPC = module.cwrap("hyperdb_wasm_arrow_insert_finish", "pointer", ["number", "string", "string", "number"]);
       this.pollQuery = module.cwrap("hyperdb_wasm_query_poll", "pointer", ["number"]);
       this.cancelQuery = module.cwrap("hyperdb_wasm_query_cancel", "pointer", ["number"]);
       this.releaseQuery = module.cwrap("hyperdb_wasm_query_release", "pointer", ["number"]);
@@ -44,6 +48,19 @@
           return this.disconnect(request.connection);
         case "startQuery":
           return this.startQuery(request.connection, request.sql);
+        case "appendArrowIPC": {
+          const pointer = safeInteger(this.module._malloc(request.chunk.byteLength || 1), "allocation pointer");
+          if (pointer === 0) throw new Error("wasm allocation failed");
+          try {
+            if (pointer + request.chunk.byteLength > this.module.HEAPU8.byteLength) throw new Error("allocation exceeds the wasm heap");
+            this.module.HEAPU8.set(request.chunk, pointer);
+            return this.appendArrowIPC(request.connection, pointer, request.chunk.byteLength);
+          } finally {
+            this.free(pointer);
+          }
+        }
+        case "finishArrowIPC":
+          return this.finishArrowIPC(request.connection, request.name, request.schema ?? "", Number(request.create));
         case "pollQuery":
           return this.pollQuery(request.query);
         case "cancelQuery":
@@ -102,6 +119,10 @@
         return hasKeys(value, ["id", "type", "connection"]) && Number.isSafeInteger(value.connection);
       case "startQuery":
         return hasKeys(value, ["id", "type", "connection", "sql"]) && Number.isSafeInteger(value.connection) && typeof value.sql === "string";
+      case "appendArrowIPC":
+        return hasKeys(value, ["id", "type", "connection", "chunk"]) && Number.isSafeInteger(value.connection) && value.chunk instanceof Uint8Array;
+      case "finishArrowIPC":
+        return hasKeys(value, ["id", "type", "connection", "name", "schema", "create"]) && Number.isSafeInteger(value.connection) && typeof value.name === "string" && (typeof value.schema === "string" || value.schema === null) && typeof value.create === "boolean";
       case "pollQuery":
       case "cancelQuery":
       case "releaseQuery":

package/dist/client.d.ts

@@ -13,6 +13,8 @@ export declare class HyperDBEngineClient {
     connect(): Promise<WasmResult>;
     disconnect(connection: number): Promise<WasmResult>;
     startQuery(connection: number, sql: string): Promise<WasmResult>;
+    appendArrowIPC(connection: number, chunk: Uint8Array): Promise<WasmResult>;
+    finishArrowIPC(connection: number, name: string, schema: string | null, create: boolean): Promise<WasmResult>;
     pollQuery(query: number): Promise<WasmResult>;
     cancelQuery(query: number): Promise<WasmResult>;
     releaseQuery(query: number): Promise<WasmResult>;