@hpcc-js/wasm-duckdb 1.12.0 → 1.13.0

package/package.json

@@ -1,47 +1,45 @@
 {
   "name": "@hpcc-js/wasm-duckdb",
-  "version": "1.12.0",
+  "version": "1.13.0",
   "description": "hpcc-js - WASM DuckDB",
   "type": "module",
   "exports": {
     ".": {
-      "types": "./types/src/index.d.ts",
+      "types": "./types/index.d.ts",
       "default": "./dist/index.js"
     }
   },
   "main": "./dist/index.js",
-  "types": "./types/src/index.d.ts",
+  "types": "./types/index.d.ts",
   "files": [
     "dist/**/*",
     "src/**/*",
     "types/**/*"
   ],
   "scripts": {
-    "clean": "rimraf coverage build dist dist-test types",
-    "pack-duckdb-eh-worker-node": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-node-eh.worker.cjs > ./build/duckdb-node-eh.worker.ts",
-    "pack-duckdb-eh-worker": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js > ./build/duckdb-browser-eh.worker.ts",
-    "pack-duckdb-eh": "npx -y mkdirp build && node ./utils/sfx-wasm.js ../../node_modules/@duckdb/duckdb-wasm/dist/duckdb-eh.wasm > ./build/duckdb-eh.wasm.ts",
-    "pack-duckdb": "run-p pack-duckdb-eh pack-duckdb-eh-worker pack-duckdb-eh-worker-node",
+    "clean": "rimraf coverage dist dist-test types .nyc_output",
+    "build-cpp": "cmake --build ../../build --target duckdblib",
+    "build-cpp-watch": "chokidar 'src-cpp/**.*' -c 'npm run build-cpp'",
     "gen-types": "tsc --project tsconfig.json --emitDeclarationOnly",
     "gen-types-watch": "npm run gen-types -- --watch",
     "bundle": "node esbuild.js",
     "bundle-dev": "npm run bundle -- --development",
     "bundle-watch": "npm run bundle-dev -- --watch",
     "build-dev": "run-p gen-types bundle-dev",
-    "build": "npm-run-all --serial pack-duckdb --parallel gen-types bundle",
+    "build": "run-p gen-types bundle",
     "lint-skypack": "npx -y @skypack/package-check",
-    "lint-eslint": "eslint src/**/*.ts tests/**/*.ts",
+    "lint-eslint": "eslint \"src/**/*.ts\" \"tests/**/*.ts\"",
     "lint": "run-p lint-eslint",
     "test-browser": "vitest run --project browser",
+    "test-node": "vitest run --project node",
     "test": "vitest run",
     "coverage": "vitest run --coverage",
     "update": "npx -y npm-check-updates -u -t minor",
     "update-major": "npx -y npm-check-updates -u"
   },
   "devDependencies": {
-    "@duckdb/duckdb-wasm": "1.31.0",
     "@hpcc-js/esbuild-plugins": "1.7.0",
-    "mkdirp": "3.0.1"
+    "@hpcc-js/wasm-util": "1.0.0"
   },
   "keywords": [
     "DuckDB",
@@ -57,5 +55,5 @@
   },
   "homepage": "https://hpcc-systems.github.io/hpcc-js-wasm/",
   "license": "Apache-2.0",
-  "gitHead": "3fa48df3d5a7a6c43bf4f0c8839f1f9df27080da"
+  "gitHead": "6799e024fc9186c4a4f87294862142f00628e597"
 }

package/src/duckdb.ts

@@ -1,81 +1,288 @@
-import load, { reset } from "../build/duckdb-eh.wasm.ts";
-import loadWasmWorker, { reset as resetWasmWorker } from "../build/duckdb-browser-eh.worker.ts";
-import { AsyncDuckDB, ConsoleLogger } from "@duckdb/duckdb-wasm";
+// @ts-expect-error importing from a wasm file is resolved via a custom esbuild plugin
+import load, { reset } from "../../../build/packages/duckdb/duckdblib.wasm";
+import type { MainModule, DuckDB as CPPDuckDB } from "../../../build/packages/duckdb/duckdblib.js";
+import { MainModuleEx } from "@hpcc-js/wasm-util";
+
+let g_duckdb: Promise<DuckDB>;
+const textEncoder = new TextEncoder();
 
 /**
- * DuckDB WASM library, a in-process SQL OLAP Database Management System..
+ * DuckDB WASM library - an in-process SQL OLAP Database Management System.
+ * 
+ * DuckDB is designed for OLAP (Online Analytical Processing) workloads and provides:
+ * - Full SQL support with rich analytical functions
+ * - In-process execution (no server required)
+ * - Efficient columnar storage
+ * - Support for reading CSV, JSON, and other formats
+ * - Transaction management
+ * - Prepared statements with parameter binding
  * 
- * See [DuckDB](https://github.com/duckdb/duckdb) for more details.
- *
+ * @example Basic Query
  * ```ts
  * import { DuckDB } from "@hpcc-js/wasm-duckdb";
  * 
- * let duckdb = await DuckDB.load();
- * const c = await duckdb.db.connect();
+ * const duckdb = await DuckDB.load();
+ * const connection = duckdb.connect();
+ * 
+ * const result = connection.query("SELECT 'Hello, DuckDB!' AS message");
+ * console.log(result.getValue(0, 0)); // "Hello, DuckDB!"
+ * 
+ * result.delete();
+ * connection.delete();
+ * ```
+ * 
+ * @example Working with JSON Files
+ * ```ts
+ * const duckdb = await DuckDB.load();
+ * const connection = duckdb.connect();
  * 
  * const data = [
- *     { "col1": 1, "col2": "foo" },
- *     { "col1": 2, "col2": "bar" },
+ *     { id: 1, name: "Alice", age: 30 },
+ *     { id: 2, name: "Bob", age: 25 }
  * ];
- * await duckdb.db.registerFileText("rows.json", JSON.stringify(data));
- * await c.insertJSONFromPath('rows.json', { name: 'rows' });
- * 
- * const arrowResult = await c.query("SELECT * FROM read_json_auto('rows.json')");
- * const result = arrowResult.toArray().map((row) => row.toJSON());
- * expect(result.length).to.equal(data.length);
- * for (let i = 0; i < result.length; i++) {
- *     expect(result[i].col2).to.equal(data[i].col2);
+ * duckdb.registerFileString("users.json", JSON.stringify(data));
+ * 
+ * const result = connection.query("SELECT * FROM read_json_auto('users.json') WHERE age > 26");
+ * console.log(result.toJSON()); // [{"id":1,"name":"Alice","age":30}]
+ * 
+ * result.delete();
+ * connection.delete();
+ * ```
+ * 
+ * @example Prepared Statements
+ * ```ts
+ * const connection = duckdb.connect();
+ * connection.query("CREATE TABLE users (id INTEGER, name VARCHAR, age INTEGER)").delete();
+ * 
+ * const insertStmt = connection.prepare("INSERT INTO users VALUES (?, ?, ?)");
+ * insertStmt.execute([1, "Alice", 30]).delete();
+ * insertStmt.execute([2, "Bob", 25]).delete();
+ * insertStmt.delete();
+ * 
+ * const queryStmt = connection.prepare("SELECT * FROM users WHERE age > ?");
+ * const result = queryStmt.execute([26]);
+ * console.log(result.toJSON());
+ * 
+ * result.delete();
+ * queryStmt.delete();
+ * connection.delete();
+ * ```
+ * 
+ * @example Transaction Management
+ * ```ts
+ * const connection = duckdb.connect();
+ * 
+ * connection.setAutoCommit(false);
+ * connection.beginTransaction();
+ * 
+ * try {
+ *     connection.query("UPDATE accounts SET balance = balance - 200 WHERE id = 1").delete();
+ *     connection.query("UPDATE accounts SET balance = balance + 200 WHERE id = 2").delete();
+ *     connection.commit();
+ *     console.log("Transfer successful");
+ * } catch (error) {
+ *     connection.rollback();
+ *     console.error("Transfer failed");
  * }
  * 
- * c.close();
+ * connection.delete();
  * ```
+ * 
+ * @see [DuckDB Documentation](https://duckdb.org/docs/)
+ * @see [DuckDB GitHub](https://github.com/duckdb/duckdb)
  */
-export class DuckDB {
 
-    db: AsyncDuckDB;
 
-    private constructor(db: AsyncDuckDB, protected _version: string) {
-        this.db = db;
+
+export class DuckDB extends MainModuleEx<MainModule> {
+
+    db: CPPDuckDB
+
+    private constructor(_module: MainModule) {
+        super(_module);
+        this.db = this._module.create()!;
+        const { FS_createPath } = this._module;
+        FS_createPath("/", "home/web_user", true, true);
     }
 
     /**
-     * Compiles and instantiates the raw wasm.
+     * Loads and initializes the DuckDB WASM module.
+     * 
+     * This method compiles and instantiates the WebAssembly module. It returns a singleton
+     * instance, so subsequent calls will return the same instance.
      * 
-     * ::: info
-     * In general WebAssembly compilation is disallowed on the main thread if the buffer size is larger than 4KB, hence forcing `load` to be asynchronous;
-     * :::
+     * **Note:** WebAssembly compilation is disallowed on the main thread if the buffer size 
+     * is larger than 4KB, which is why this method is asynchronous.
      * 
-     * @returns A promise to an instance of the DuckDB class.
+     * @returns A promise that resolves to the DuckDB instance
+     * 
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.version()); // e.g., "v1.4.3"
+     * ```
      */
     static load(): Promise<DuckDB> {
-        const workerUrl = URL.createObjectURL(
-            new Blob([new Uint8Array(loadWasmWorker())], { type: "text/javascript" })
-        );
-        const worker = new Worker(workerUrl);
-        URL.revokeObjectURL(workerUrl);
-        const logger = new ConsoleLogger();
-        const db = new AsyncDuckDB(logger, worker);
-        const wasmUrl = URL.createObjectURL(
-            new Blob([new Uint8Array(load())], { "type": "application/wasm" })
-        );
-        return db.instantiate(wasmUrl, null).then(async () => {
-            URL.revokeObjectURL(wasmUrl);
-            return new DuckDB(db, await db.getVersion());
-        });
+        if (!g_duckdb) {
+            g_duckdb = load().then((module: any) => {
+                return new DuckDB(module)
+            });
+        }
+        return g_duckdb;
     }
 
     /**
-     * Unloades the compiled wasm instance.
+     * Unloads the compiled WASM instance.
+     * 
+     * This is primarily useful for cleanup in testing scenarios or when you need to
+     * completely reset the DuckDB instance.
+     * 
+     * @example
+     * ```ts
+     * DuckDB.unload();
+     * // The next call to DuckDB.load() will create a fresh instance
+     * ```
      */
     static unload() {
-        resetWasmWorker();
         reset();
     }
 
     /**
-     * @returns The DuckDB version
+     * Returns the DuckDB version string.
+     * 
+     * @returns The version string (e.g., "v1.4.3")
+     * 
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.version()); // "v1.4.3"
+     * ```
      */
     version(): string {
-        return this._version;
+        return this._module.libraryVersion();
+    }
+
+    /**
+     * Creates a new database connection.
+     * 
+     * Each connection maintains its own transaction state and can execute queries
+     * independently. Always call `.delete()` on the connection when done to free resources.
+     * 
+     * @returns A new Connection instance
+     * 
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * const connection = duckdb.connect();
+     * 
+     * const result = connection.query("SELECT 42 AS answer");
+     * console.log(result.getValue(0, 0)); // 42
+     * 
+     * result.delete();
+     * connection.delete(); // Important: clean up!
+     * ```
+     */
+    connect() {
+        return this.db.connect()!;
     }
+
+    /**
+     * Returns the number of threads available to DuckDB.
+     * 
+     * @returns The number of threads
+     * 
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.numberOfThreads()); // 1
+     * ```
+     */
+    numberOfThreads(): number {
+        return Number(this.db.numberOfThreads());
+    }
+
+    /**
+     * Registers a binary file in the virtual file system.
+     * 
+     * This allows DuckDB to read files as if they were on disk, which is useful for
+     * reading CSV, JSON, Parquet, and other file formats using DuckDB's file readers.
+     * 
+     * The path is normalized to remove leading slashes and directories are created
+     * automatically as needed.
+     * 
+     * @param path - The path where the file should be accessible (e.g., "data/users.csv")
+     * @param content - The file content as a Uint8Array
+     * 
+     * @example
+     * ```ts
+     * const csvData = "id,name\n1,Alice\n2,Bob";
+     * const bytes = new TextEncoder().encode(csvData);
+     * duckdb.registerFile("data.csv", bytes);
+     * 
+     * const connection = duckdb.connect();
+     * const result = connection.query("SELECT * FROM read_csv_auto('data.csv')");
+     * console.log(result.rowCount()); // 2
+     * result.delete();
+     * connection.delete();
+     * ```
+     */
+    registerFile(path: string, content: Uint8Array): void {
+        const normalizedPath = path.startsWith("/") ? path.slice(1) : path;
+        const split = normalizedPath.lastIndexOf("/");
+        const dir = split > 0 ? normalizedPath.substring(0, split) : "/";
+        if (dir.length > 1 && this._module.FS_createPath) {
+            this._module.FS_createPath(dir, true, true);
+        }
+        this._module.FS_createDataFile(normalizedPath, undefined, content, true, true, true);
+    }
+
+    /**
+     * Registers a text file in the virtual file system.
+     * 
+     * This is a convenience method that encodes the string content to UTF-8 bytes
+     * and calls {@link registerFile}. Useful for registering JSON, CSV, or other
+     * text-based data files.
+     * 
+     * @param fileName - The path where the file should be accessible
+     * @param content - The file content as a string
+     * 
+     * @example Register JSON data
+     * ```ts
+     * const users = [
+     *     { id: 1, name: "Alice", age: 30 },
+     *     { id: 2, name: "Bob", age: 25 }
+     * ];
+     * duckdb.registerFileString("users.json", JSON.stringify(users));
+     * 
+     * const connection = duckdb.connect();
+     * const result = connection.query(`
+     *     SELECT name, age 
+     *     FROM read_json_auto('users.json') 
+     *     WHERE age > 26
+     * `);
+     * console.log(result.toJSON()); // [{"name":"Alice","age":30}]
+     * result.delete();
+     * connection.delete();
+     * ```
+     * 
+     * @example Register CSV data
+     * ```ts
+     * const csvData = `id,product,price
+     * 1,Laptop,999.99
+     * 2,Mouse,29.99`;
+     * 
+     * duckdb.registerFileString("products.csv", csvData);
+     * 
+     * const connection = duckdb.connect();
+     * const result = connection.query("SELECT * FROM read_csv_auto('products.csv')");
+     * result.print();
+     * result.delete();
+     * connection.delete();
+     * ```
+     */
+    registerFileString(fileName: string, content: string): void {
+        const encoded = textEncoder.encode(content);
+        this.registerFile(fileName, encoded);
+    }
+
 }

package/types/duckdb.d.ts

@@ -0,0 +1,238 @@
+import type { MainModule, DuckDB as CPPDuckDB } from "../../../build/packages/duckdb/duckdblib.js";
+import { MainModuleEx } from "@hpcc-js/wasm-util";
+/**
+ * DuckDB WASM library - an in-process SQL OLAP Database Management System.
+ *
+ * DuckDB is designed for OLAP (Online Analytical Processing) workloads and provides:
+ * - Full SQL support with rich analytical functions
+ * - In-process execution (no server required)
+ * - Efficient columnar storage
+ * - Support for reading CSV, JSON, and other formats
+ * - Transaction management
+ * - Prepared statements with parameter binding
+ *
+ * @example Basic Query
+ * ```ts
+ * import { DuckDB } from "@hpcc-js/wasm-duckdb";
+ *
+ * const duckdb = await DuckDB.load();
+ * const connection = duckdb.connect();
+ *
+ * const result = connection.query("SELECT 'Hello, DuckDB!' AS message");
+ * console.log(result.getValue(0, 0)); // "Hello, DuckDB!"
+ *
+ * result.delete();
+ * connection.delete();
+ * ```
+ *
+ * @example Working with JSON Files
+ * ```ts
+ * const duckdb = await DuckDB.load();
+ * const connection = duckdb.connect();
+ *
+ * const data = [
+ *     { id: 1, name: "Alice", age: 30 },
+ *     { id: 2, name: "Bob", age: 25 }
+ * ];
+ * duckdb.registerFileString("users.json", JSON.stringify(data));
+ *
+ * const result = connection.query("SELECT * FROM read_json_auto('users.json') WHERE age > 26");
+ * console.log(result.toJSON()); // [{"id":1,"name":"Alice","age":30}]
+ *
+ * result.delete();
+ * connection.delete();
+ * ```
+ *
+ * @example Prepared Statements
+ * ```ts
+ * const connection = duckdb.connect();
+ * connection.query("CREATE TABLE users (id INTEGER, name VARCHAR, age INTEGER)").delete();
+ *
+ * const insertStmt = connection.prepare("INSERT INTO users VALUES (?, ?, ?)");
+ * insertStmt.execute([1, "Alice", 30]).delete();
+ * insertStmt.execute([2, "Bob", 25]).delete();
+ * insertStmt.delete();
+ *
+ * const queryStmt = connection.prepare("SELECT * FROM users WHERE age > ?");
+ * const result = queryStmt.execute([26]);
+ * console.log(result.toJSON());
+ *
+ * result.delete();
+ * queryStmt.delete();
+ * connection.delete();
+ * ```
+ *
+ * @example Transaction Management
+ * ```ts
+ * const connection = duckdb.connect();
+ *
+ * connection.setAutoCommit(false);
+ * connection.beginTransaction();
+ *
+ * try {
+ *     connection.query("UPDATE accounts SET balance = balance - 200 WHERE id = 1").delete();
+ *     connection.query("UPDATE accounts SET balance = balance + 200 WHERE id = 2").delete();
+ *     connection.commit();
+ *     console.log("Transfer successful");
+ * } catch (error) {
+ *     connection.rollback();
+ *     console.error("Transfer failed");
+ * }
+ *
+ * connection.delete();
+ * ```
+ *
+ * @see [DuckDB Documentation](https://duckdb.org/docs/)
+ * @see [DuckDB GitHub](https://github.com/duckdb/duckdb)
+ */
+export declare class DuckDB extends MainModuleEx<MainModule> {
+    db: CPPDuckDB;
+    private constructor();
+    /**
+     * Loads and initializes the DuckDB WASM module.
+     *
+     * This method compiles and instantiates the WebAssembly module. It returns a singleton
+     * instance, so subsequent calls will return the same instance.
+     *
+     * **Note:** WebAssembly compilation is disallowed on the main thread if the buffer size
+     * is larger than 4KB, which is why this method is asynchronous.
+     *
+     * @returns A promise that resolves to the DuckDB instance
+     *
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.version()); // e.g., "v1.4.3"
+     * ```
+     */
+    static load(): Promise<DuckDB>;
+    /**
+     * Unloads the compiled WASM instance.
+     *
+     * This is primarily useful for cleanup in testing scenarios or when you need to
+     * completely reset the DuckDB instance.
+     *
+     * @example
+     * ```ts
+     * DuckDB.unload();
+     * // The next call to DuckDB.load() will create a fresh instance
+     * ```
+     */
+    static unload(): void;
+    /**
+     * Returns the DuckDB version string.
+     *
+     * @returns The version string (e.g., "v1.4.3")
+     *
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.version()); // "v1.4.3"
+     * ```
+     */
+    version(): string;
+    /**
+     * Creates a new database connection.
+     *
+     * Each connection maintains its own transaction state and can execute queries
+     * independently. Always call `.delete()` on the connection when done to free resources.
+     *
+     * @returns A new Connection instance
+     *
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * const connection = duckdb.connect();
+     *
+     * const result = connection.query("SELECT 42 AS answer");
+     * console.log(result.getValue(0, 0)); // 42
+     *
+     * result.delete();
+     * connection.delete(); // Important: clean up!
+     * ```
+     */
+    connect(): import("../../../build/packages/duckdb/duckdblib.js").Connection;
+    /**
+     * Returns the number of threads available to DuckDB.
+     *
+     * @returns The number of threads
+     *
+     * @example
+     * ```ts
+     * const duckdb = await DuckDB.load();
+     * console.log(duckdb.numberOfThreads()); // 1
+     * ```
+     */
+    numberOfThreads(): number;
+    /**
+     * Registers a binary file in the virtual file system.
+     *
+     * This allows DuckDB to read files as if they were on disk, which is useful for
+     * reading CSV, JSON, Parquet, and other file formats using DuckDB's file readers.
+     *
+     * The path is normalized to remove leading slashes and directories are created
+     * automatically as needed.
+     *
+     * @param path - The path where the file should be accessible (e.g., "data/users.csv")
+     * @param content - The file content as a Uint8Array
+     *
+     * @example
+     * ```ts
+     * const csvData = "id,name\n1,Alice\n2,Bob";
+     * const bytes = new TextEncoder().encode(csvData);
+     * duckdb.registerFile("data.csv", bytes);
+     *
+     * const connection = duckdb.connect();
+     * const result = connection.query("SELECT * FROM read_csv_auto('data.csv')");
+     * console.log(result.rowCount()); // 2
+     * result.delete();
+     * connection.delete();
+     * ```
+     */
+    registerFile(path: string, content: Uint8Array): void;
+    /**
+     * Registers a text file in the virtual file system.
+     *
+     * This is a convenience method that encodes the string content to UTF-8 bytes
+     * and calls {@link registerFile}. Useful for registering JSON, CSV, or other
+     * text-based data files.
+     *
+     * @param fileName - The path where the file should be accessible
+     * @param content - The file content as a string
+     *
+     * @example Register JSON data
+     * ```ts
+     * const users = [
+     *     { id: 1, name: "Alice", age: 30 },
+     *     { id: 2, name: "Bob", age: 25 }
+     * ];
+     * duckdb.registerFileString("users.json", JSON.stringify(users));
+     *
+     * const connection = duckdb.connect();
+     * const result = connection.query(`
+     *     SELECT name, age
+     *     FROM read_json_auto('users.json')
+     *     WHERE age > 26
+     * `);
+     * console.log(result.toJSON()); // [{"name":"Alice","age":30}]
+     * result.delete();
+     * connection.delete();
+     * ```
+     *
+     * @example Register CSV data
+     * ```ts
+     * const csvData = `id,product,price
+     * 1,Laptop,999.99
+     * 2,Mouse,29.99`;
+     *
+     * duckdb.registerFileString("products.csv", csvData);
+     *
+     * const connection = duckdb.connect();
+     * const result = connection.query("SELECT * FROM read_csv_auto('products.csv')");
+     * result.print();
+     * result.delete();
+     * connection.delete();
+     * ```
+     */
+    registerFileString(fileName: string, content: string): void;
+}

package/types/build/duckdb-browser-eh.worker.d.ts

@@ -1,2 +0,0 @@
-export default function (): Uint8Array<ArrayBufferLike>;
-export declare function reset(): void;

package/types/build/duckdb-eh.wasm.d.ts

@@ -1,2 +0,0 @@
-export default function (): Uint8Array<ArrayBufferLike>;
-export declare function reset(): void;