@uwdata/mosaic-duckdb 0.21.1 → 0.23.0

package/bin/to-arrow.js

@@ -21,4 +21,5 @@ output.on('error', (error) => {
 });
 
 // write arrow bytes to output
-output.end(buf);
+for (const chunk of buf) output.write(chunk);
+output.end();

package/dist/src/DuckDB.d.ts

@@ -1,22 +1,68 @@
 export class DuckDB {
-    constructor(path?: string, config?: {}, initStatements?: string);
-    db: duckdb.Database;
-    con: duckdb.Connection;
-    _init: Promise<any>;
-    close(): Promise<any>;
-    prepare(sql: any): DuckDBStatement;
-    exec(sql: any): Promise<any>;
-    query(sql: any): Promise<any>;
-    arrowBuffer(sql: any): Promise<any>;
+    /**
+     * @param {string} path
+     * @param {Record<string, string>} config
+     * @param {string} initStatements
+     */
+    constructor(path?: string, config?: Record<string, string>, initStatements?: string);
+    /** @type {Promise<void>} */
+    _init: Promise<void>;
+    /** @type {DuckDBInstance | undefined} */
+    db: DuckDBInstance | undefined;
+    /** @type {DuckDBConnection | undefined} */
+    con: DuckDBConnection | undefined;
+    /**
+     * @param {string} path
+     * @param {Record<string, string>} config
+     * @param {string} initStatements
+     */
+    _initialize(path: string, config: Record<string, string>, initStatements: string): Promise<void>;
+    close(): void;
+    /**
+     * @param {string} sql
+     * @returns {Promise<DuckDBStatement>}
+     */
+    prepare(sql: string): Promise<DuckDBStatement>;
+    /**
+     * @param {string | { toString(): string }} sql
+     * @returns {Promise<this>}
+     */
+    exec(sql: string | {
+        toString(): string;
+    }): Promise<this>;
+    /**
+     * @param {string} sql
+     * @returns {Promise<Record<string, Json>[]>}
+     */
+    query(sql: string): Promise<Record<string, Json>[]>;
+    /**
+     * @param {string} sql
+     * @returns {Promise<Uint8Array[]>}
+     */
+    arrowBuffer(sql: string): Promise<Uint8Array[]>;
 }
 export class DuckDBStatement {
-    constructor(statement: any);
-    statement: any;
+    /** @param {DuckDBPreparedStatement} statement */
+    constructor(statement: DuckDBPreparedStatement);
+    /** @type {DuckDBPreparedStatement} */
+    statement: DuckDBPreparedStatement;
     finalize(): void;
-    run(params: any): void;
-    exec(params: any): Promise<any>;
-    query(params: any): Promise<any>;
-    arrowBuffer(params: any): Promise<any>;
+    /** @param {DuckDBValue[]} [params] */
+    run(params?: DuckDBValue[]): Promise<void>;
+    /**
+     * @param {DuckDBValue[]} [params]
+     * @returns {Promise<this>}
+     */
+    exec(params?: DuckDBValue[]): Promise<this>;
+    /**
+     * @param {DuckDBValue[]} [params]
+     * @returns {Promise<Record<string, Json>[]>}
+     */
+    query(params?: DuckDBValue[]): Promise<Record<string, Json>[]>;
 }
-import duckdb from 'duckdb';
+import { DuckDBInstance } from '@duckdb/node-api';
+import type { DuckDBConnection } from '@duckdb/node-api';
+import type { Json } from '@duckdb/node-api';
+import type { DuckDBPreparedStatement } from '@duckdb/node-api';
+import type { DuckDBValue } from '@duckdb/node-api';
 //# sourceMappingURL=DuckDB.d.ts.map

package/dist/src/DuckDB.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"DuckDB.d.ts","sourceRoot":"","sources":["../../src/DuckDB.js"],"names":[],"mappings":"AAaA;IACE,iEASC;IAJC,oBAA2C;IAC3C,uBAA4B;IAE5B,oBAAsC;IAGxC,sBAUC;IAED,mCAEC;IAED,6BAUC;IAED,8BAUC;IAED,oCAUC;CACF;AAED;IACE,4BAEC;IADC,eAA0B;IAG5B,iBAEC;IAED,uBAEC;IAED,gCAUC;IAED,iCAUC;IAED,uCAUC;CACF;mBA9HkB,QAAQ"}
+{"version":3,"file":"DuckDB.d.ts","sourceRoot":"","sources":["../../src/DuckDB.js"],"names":[],"mappings":"AAcA;IAQE;;;;OAIG;IACH,mBAJW,MAAM,WACN,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,mBACtB,MAAM,EAQhB;IAlBD,4BAA4B;IAC5B,OADW,OAAO,CAAC,IAAI,CAAC,CAClB;IACN,yCAAyC;IACzC,IADW,cAAc,GAAG,SAAS,CAClC;IACH,2CAA2C;IAC3C,KADW,gBAAgB,GAAG,SAAS,CACnC;IAeJ;;;;OAIG;IACH,kBAJW,MAAM,UACN,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,kBACtB,MAAM,iBAMhB;IAED,cAGC;IAED;;;OAGG;IACH,aAHW,MAAM,GACJ,OAAO,CAAC,eAAe,CAAC,CAMpC;IAED;;;OAGG;IACH,UAHW,MAAM,GAAG;QAAE,QAAQ,IAAI,MAAM,CAAA;KAAE,GAC7B,OAAO,CAAC,IAAI,CAAC,CAMzB;IAED;;;OAGG;IACH,WAHW,MAAM,GACJ,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAM3C;IAED;;;OAGG;IACH,iBAHW,MAAM,GACJ,OAAO,CAAC,UAAU,EAAE,CAAC,CAQjC;CAEF;AAED;IAIE,iDAAiD;IACjD,uBADY,uBAAuB,EAGlC;IAND,sCAAsC;IACtC,WADW,uBAAuB,CACxB;IAOV,iBAEC;IAED,sCAAsC;IACtC,aADY,WAAW,EAAE,iBAIxB;IAED;;;OAGG;IACH,cAHW,WAAW,EAAE,GACX,OAAO,CAAC,IAAI,CAAC,CAMzB;IAED;;;OAGG;IACH,eAHW,WAAW,EAAE,GACX,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,CAM3C;CACF;+BArI8B,kBAAkB;sCAEiC,kBAAkB;0BAAlB,kBAAkB;6CAAlB,kBAAkB;iCAAlB,kBAAkB"}

package/dist/src/data-server.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"data-server.d.ts","sourceRoot":"","sources":["../../src/data-server.js"],"names":[],"mappings":"AAOA;;;;;yEAgBC;AA6CD,yDAqBgB,QAAG,EAAE,SAAI,mBAwCxB;AAwBD;;;;;EAmBC;iBA5KgB,WAAW"}
+{"version":3,"file":"data-server.d.ts","sourceRoot":"","sources":["../../src/data-server.js"],"names":[],"mappings":"AAOA;;;;;yEAgBC;AA6CD,yDAqBgB,QAAG,EAAE,SAAI,mBAwCxB;AAyBD;;;;;EAmBC;iBA7KgB,WAAW"}

package/dist/src/load/arrow.d.ts

@@ -1,2 +1,2 @@
-export function loadArrow(db: any, tableName: any, buffer: any): Promise<any>;
+export function loadArrow(db: any, tableName: any, buffer: any): Promise<void>;
 //# sourceMappingURL=arrow.d.ts.map

package/dist/src/load/arrow.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow.d.ts","sourceRoot":"","sources":["../../../src/load/arrow.js"],"names":[],"mappings":"AAEA,8EAWC"}
+{"version":3,"file":"arrow.d.ts","sourceRoot":"","sources":["../../../src/load/arrow.js"],"names":[],"mappings":"AAKA,+EAWC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-duckdb",
-  "version": "0.21.1",
+  "version": "0.23.0",
   "description": "A Promise-based DuckDB API and Node.js data server.",
   "keywords": [
     "duckdb",
@@ -36,12 +36,16 @@
     "lint": "eslint src test",
     "server": "node bin/run-server.js",
     "test": "vitest run",
-    "prepublishOnly": "npm run test && npm run lint && tsc --build"
+    "prepublishOnly": "pnpm run test && pnpm run lint && tsc --build"
   },
   "dependencies": {
-    "@uwdata/mosaic-sql": "^0.21.1",
-    "duckdb": "~1.4.1",
-    "ws": "^8.18.3"
+    "@duckdb/node-api": "~1.4.4-r.3",
+    "@uwdata/mosaic-sql": "^0.23.0",
+    "ws": "^8.20.0"
   },
-  "gitHead": "296762423ac032193344646ad4f10cf604ca69a2"
+  "devDependencies": {
+    "@types/node": "^25.5.0",
+    "@uwdata/flechette": "^2.3.0"
+  },
+  "gitHead": "8d153a15b9b1c69130d5384ecd59aa634e3f90b3"
 }

package/src/DuckDB.js

@@ -1,5 +1,6 @@
-import duckdb from 'duckdb';
-import { mergeBuffers } from './merge-buffers.js';
+import { DuckDBInstance } from '@duckdb/node-api';
+
+/** @import { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue, Json } from '@duckdb/node-api' */
 
 const TEMP_DIR = '.duckdb';
 
@@ -12,116 +13,122 @@ const DEFAULT_INIT_STATEMENTS = [
 ].join(';\n');
 
 export class DuckDB {
+  /** @type {Promise<void>} */
+  _init;
+  /** @type {DuckDBInstance | undefined} */
+  db;
+  /** @type {DuckDBConnection | undefined} */
+  con;
+
+  /**
+   * @param {string} path
+   * @param {Record<string, string>} config
+   * @param {string} initStatements
+   */
   constructor(
     path = ':memory:',
     config = {},
     initStatements = DEFAULT_INIT_STATEMENTS
   ) {
-    this.db = new duckdb.Database(path, config);
-    this.con = this.db.connect();
-    // store initialization promise so that we can wait for it
-    this._init = this.exec(initStatements);
+    this._init = this._initialize(path, config, initStatements);
+  }
+
+  /**
+   * @param {string} path
+   * @param {Record<string, string>} config
+   * @param {string} initStatements
+   */
+  async _initialize(path, config, initStatements) {
+    this.db = await DuckDBInstance.create(path, config);
+    this.con = await this.db.connect();
+    await this.con.run(initStatements);
   }
 
   close() {
-    return new Promise((resolve, reject) => {
-      this.db.close((err) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(this);
-        }
-      });
-    });
+    this.con?.closeSync();
+    this.db?.closeSync();
   }
 
-  prepare(sql) {
-    return new DuckDBStatement(this.con.prepare(sql));
+  /**
+   * @param {string} sql
+   * @returns {Promise<DuckDBStatement>}
+   */
+  async prepare(sql) {
+    await this._init;
+    const stmt = await this.con.prepare(sql);
+    return new DuckDBStatement(stmt);
   }
 
-  exec(sql) {
-    return new Promise((resolve, reject) => {
-      this.con.exec(sql, (err) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(this);
-        }
-      });
-    });
+  /**
+   * @param {string | { toString(): string }} sql
+   * @returns {Promise<this>}
+   */
+  async exec(sql) {
+    await this._init;
+    await this.con.run(String(sql));
+    return this;
   }
 
-  query(sql) {
-    return new Promise((resolve, reject) => {
-      this.con.all(sql, (err, result) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(result);
-        }
-      });
-    });
+  /**
+   * @param {string} sql
+   * @returns {Promise<Record<string, Json>[]>}
+   */
+  async query(sql) {
+    await this._init;
+    const reader = await this.con.runAndReadAll(sql);
+    return reader.getRowObjectsJson();
   }
 
-  arrowBuffer(sql) {
-    return new Promise((resolve, reject) => {
-      this.con.arrowIPCAll(sql, (err, result) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(mergeBuffers(result));
-        }
-      });
-    });
+  /**
+   * @param {string} sql
+   * @returns {Promise<Uint8Array[]>}
+   */
+  async arrowBuffer(sql) {
+    await this._init;
+    const reader = await this.con.runAndReadAll(
+      `SELECT * FROM to_arrow_ipc((${sql}))`
+    );
+    return /** @type {Uint8Array[]} */ (reader.getColumnsJS()[0]) ?? [];
   }
+
 }
 
 export class DuckDBStatement {
+  /** @type {DuckDBPreparedStatement} */
+  statement;
+
+  /** @param {DuckDBPreparedStatement} statement */
   constructor(statement) {
     this.statement = statement;
   }
 
   finalize() {
-    this.statement.finalize();
-  }
-
-  run(params) {
-    this.statement.run(...params);
+    this.statement.destroySync();
   }
 
-  exec(params) {
-    return new Promise((resolve, reject) => {
-      this.statement.run(...params, (err) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(this);
-        }
-      });
-    });
+  /** @param {DuckDBValue[]} [params] */
+  async run(params) {
+    if (params?.length) this.statement.bind(params);
+    await this.statement.run();
   }
 
-  query(params) {
-    return new Promise((resolve, reject) => {
-      this.statement.all(...params, (err, result) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(result);
-        }
-      });
-    });
+  /**
+   * @param {DuckDBValue[]} [params]
+   * @returns {Promise<this>}
+   */
+  async exec(params) {
+    if (params?.length) this.statement.bind(params);
+    await this.statement.run();
+    return this;
   }
 
-  arrowBuffer(params) {
-    return new Promise((resolve, reject) => {
-      this.statement.arrowIPCAll(...params, (err, result) => {
-        if (err) {
-          reject(err);
-        } else {
-          resolve(mergeBuffers(result));
-        }
-      });
-    });
+  /**
+   * @param {DuckDBValue[]} [params]
+   * @returns {Promise<Record<string, Json>[]>}
+   */
+  async query(params) {
+    if (params?.length) this.statement.bind(params);
+    const reader = await this.statement.runAndReadAll();
+    return reader.getRowObjectsJson();
   }
 }

package/src/data-server.js

@@ -133,7 +133,8 @@ function httpResponse(res) {
   return {
     arrow(data) {
       res.setHeader('Content-Type', 'application/vnd.apache.arrow.stream');
-      res.end(data);
+      for (const chunk of data) res.write(chunk);
+      res.end();
     },
     json(data) {
       res.setHeader('Content-Type', 'application/json');
@@ -157,7 +158,7 @@ export function socketResponse(ws) {
 
   return {
     arrow(data) {
-      ws.send(data, BINARY);
+      ws.send(Buffer.concat(data), BINARY);
     },
     json(data) {
       ws.send(JSON.stringify(data), STRING);

package/src/load/arrow.js

@@ -1,14 +1,17 @@
-import { readFile } from 'node:fs/promises';
+import { readFile, writeFile, unlink } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { randomBytes } from 'node:crypto';
 
 export async function loadArrow(db, tableName, buffer) {
-  const arrowData = ArrayBuffer.isView(buffer) ? buffer : await readFile(buffer);
-  return new Promise((resolve, reject) => {
-    db.con.register_buffer(tableName, [arrowData], true, err => {
-      if (err) {
-        console.error(err);
-        reject(err);
-      }
-      resolve();
-    });
-  });
+  const arrowData = Array.isArray(buffer) ? Buffer.concat(buffer)
+    : ArrayBuffer.isView(buffer) ? Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength)
+    : await readFile(buffer);
+  const tempFile = join(tmpdir(), `mosaic_${randomBytes(8).toString('hex')}.arrow`);
+  await writeFile(tempFile, arrowData);
+  try {
+    await db.exec(`CREATE OR REPLACE TABLE "${tableName}" AS SELECT * FROM '${tempFile}'`);
+  } finally {
+    await unlink(tempFile).catch(() => {});
+  }
 }

package/src/merge-buffers.js

@@ -1,13 +0,0 @@
-export function mergeBuffers(buffers) {
-  const len = buffers.reduce((a, b) => a + (b ? b.length : 0), 0);
-  const buf = new Uint8Array(len);
-
-  for (let i = 0, offset = 0; i < buffers.length; ++i) {
-    if (buffers[i]) {
-      buf.set(buffers[i], offset);
-      offset += buffers[i].length;
-    }
-  }
-
-  return buf;
-}