@uwdata/mosaic-duckdb 0.16.2 → 0.18.0

package/LICENSE

@@ -1,6 +1,6 @@
 BSD 3-Clause License
 
-Copyright (c) 2023, UW Interactive Data Lab
+Copyright (c) 2023-2025, UW Interactive Data Lab
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:

package/README.md

@@ -4,6 +4,6 @@
 
 A Promise-based Node.js API to DuckDB, along with a data server that supports transfer of [Apache Arrow](https://arrow.apache.org/) and JSON data over either Web Sockets or HTTP.
 
-_Warning_: Due to persistent quality issues involving the Node.js DuckDB client and Arrow extension, we recommend using the Python-based [`duckdb-server`](https://github.com/uwdata/mosaic/tree/main/packages/duckdb-server) package instead. However, we retain this package for both backwards compatibility and potential future use as quality issues improve.
+_Warning_: Due to persistent quality issues involving the Node.js DuckDB client and Arrow extension, we recommend using the Python-based [`duckdb-server`](https://github.com/uwdata/mosaic/tree/main/packages/server/duckdb-server) package instead. However, we retain this package for both backwards compatibility and potential future use as quality issues improve.
 
-_Note:_ This package provides a local DuckDB server for Node.js. To instead use DuckDB-WASM in the browser, use the `wasmConnector` in the [`mosaic-core`](https://github.com/uwdata/mosaic/tree/main/packages/mosaic-core) package.
+_Note:_ This package provides a local DuckDB server for Node.js. To instead use DuckDB-WASM in the browser, use the `wasmConnector` in the [`mosaic-core`](https://github.com/uwdata/mosaic/tree/main/packages/mosaic/mosaic-core) package.

package/dist/src/Cache.d.ts

@@ -0,0 +1,22 @@
+export function cacheKey(hashable: any, type: any): string;
+export class Cache {
+    constructor({ max, dir, ttl }: {
+        max?: number;
+        dir?: string;
+        ttl?: number;
+    });
+    cache: Map<any, any>;
+    max: number;
+    dir: string;
+    ttl: number;
+    has(key: any): boolean;
+    delete(key: any): boolean;
+    get(key: any): any;
+    set(key: any, data: any, { persist, ttl }?: {
+        persist?: boolean;
+        ttl?: number;
+    }): this;
+    shouldEvict(): boolean;
+    evict(): void;
+}
+//# sourceMappingURL=Cache.d.ts.map

package/dist/src/Cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"Cache.d.ts","sourceRoot":"","sources":["../../src/Cache.js"],"names":[],"mappings":"AAOA,2DAEC;AAaD;IACE;;;;OAUC;IALC,qBAAoB;IACpB,YAAc;IACd,YAAc;IACd,YAAc;IAIhB,uBAEC;IAED,0BAMC;IAED,mBAEC;IAED;;;aAMC;IAED,uBAEC;IAED,cAyBC;CACF"}

package/dist/src/DuckDB.d.ts

@@ -0,0 +1,21 @@
+export class DuckDB {
+    constructor(path?: string, config?: {}, initStatements?: string);
+    db: duckdb.Database;
+    con: duckdb.Connection;
+    close(): Promise<any>;
+    prepare(sql: any): DuckDBStatement;
+    exec(sql: any): Promise<any>;
+    query(sql: any): Promise<any>;
+    arrowBuffer(sql: any): Promise<any>;
+}
+export class DuckDBStatement {
+    constructor(statement: any);
+    statement: any;
+    finalize(): void;
+    run(params: any): void;
+    exec(params: any): Promise<any>;
+    query(params: any): Promise<any>;
+    arrowBuffer(params: any): Promise<any>;
+}
+import duckdb from 'duckdb';
+//# sourceMappingURL=DuckDB.d.ts.map

package/dist/src/DuckDB.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"DuckDB.d.ts","sourceRoot":"","sources":["../../src/DuckDB.js"],"names":[],"mappings":"AAaA;IACE,iEAQC;IAHC,oBAA2C;IAC3C,uBAA4B;IAI9B,sBAUC;IAED,mCAEC;IAED,6BAUC;IAED,8BAUC;IAED,oCAUC;CACF;AAED;IACE,4BAEC;IADC,eAA0B;IAG5B,iBAEC;IAED,uBAEC;IAED,gCAUC;IAED,iCAUC;IAED,uCAUC;CACF;mBA7HkB,QAAQ"}

package/dist/src/data-server.d.ts

@@ -0,0 +1,15 @@
+export function dataServer(db: any, { cache, rest, socket, port }?: {
+    cache?: boolean;
+    rest?: boolean;
+    socket?: boolean;
+    port?: number;
+}): http.Server<typeof http.IncomingMessage, typeof http.ServerResponse>;
+export function queryHandler(db: any, queryCache: any): (res: any, data: any) => Promise<void>;
+export function socketResponse(ws: any): {
+    arrow(data: any): void;
+    json(data: any): void;
+    done(): void;
+    error(err: any): void;
+};
+import http from 'node:http';
+//# sourceMappingURL=data-server.d.ts.map

package/dist/src/data-server.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"data-server.d.ts","sourceRoot":"","sources":["../../src/data-server.js"],"names":[],"mappings":"AAOA;;;;;yEAgBC;AA6CD,yDAqBgB,QAAG,EAAE,SAAI,mBAwCxB;AAwBD;;;;;EAmBC;iBA5KgB,WAAW"}

package/dist/src/index.d.ts

@@ -0,0 +1,8 @@
+export { DuckDB } from "./DuckDB.js";
+export { Cache } from "./Cache.js";
+export { loadArrow } from "./load/arrow.js";
+export { loadCSV } from "./load/csv.js";
+export { loadJSON } from "./load/json.js";
+export { loadParquet } from "./load/parquet.js";
+export { dataServer, queryHandler, socketResponse } from "./data-server.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/src/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/index.js"],"names":[],"mappings":""}

package/dist/src/load/arrow.d.ts

@@ -0,0 +1,2 @@
+export function loadArrow(db: any, tableName: any, buffer: any): Promise<any>;
+//# sourceMappingURL=arrow.d.ts.map

package/dist/src/load/arrow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow.d.ts","sourceRoot":"","sources":["../../../src/load/arrow.js"],"names":[],"mappings":"AAEA,8EAWC"}

package/dist/src/load/csv.d.ts

@@ -0,0 +1,2 @@
+export function loadCSV(db: any, tableName: any, fileName: any, options?: {}): any;
+//# sourceMappingURL=csv.d.ts.map

package/dist/src/load/csv.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"csv.d.ts","sourceRoot":"","sources":["../../../src/load/csv.js"],"names":[],"mappings":"AAEA,mFAEC"}

package/dist/src/load/json.d.ts

@@ -0,0 +1,2 @@
+export function loadJSON(db: any, tableName: any, fileName: any, options?: {}): any;
+//# sourceMappingURL=json.d.ts.map

package/dist/src/load/json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json.d.ts","sourceRoot":"","sources":["../../../src/load/json.js"],"names":[],"mappings":"AAEA,oFAEC"}

package/dist/src/load/parquet.d.ts

@@ -0,0 +1,2 @@
+export function loadParquet(db: any, tableName: any, fileName: any, options?: {}): any;
+//# sourceMappingURL=parquet.d.ts.map

package/dist/src/load/parquet.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parquet.d.ts","sourceRoot":"","sources":["../../../src/load/parquet.js"],"names":[],"mappings":"AAEA,uFAEC"}

package/dist/src/merge-buffers.d.ts

@@ -0,0 +1,2 @@
+export function mergeBuffers(buffers: any): Uint8Array<any>;
+//# sourceMappingURL=merge-buffers.d.ts.map

package/dist/src/merge-buffers.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merge-buffers.d.ts","sourceRoot":"","sources":["../../src/merge-buffers.js"],"names":[],"mappings":"AAAA,4DAYC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@uwdata/mosaic-duckdb",
-  "version": "0.16.2",
+  "version": "0.18.0",
   "description": "A Promise-based DuckDB API and Node.js data server.",
   "keywords": [
     "duckdb",
@@ -13,8 +13,15 @@
   "author": "Jeffrey Heer (https://idl.uw.edu)",
   "type": "module",
   "exports": {
+    "types": "./dist/src/index.d.ts",
     "default": "./src/index.js"
   },
+  "files": [
+    "bin",
+    "dist",
+    "!dist/tsconfig.tsbuildinfo",
+    "src"
+  ],
   "repository": {
     "type": "git",
     "url": "https://github.com/uwdata/mosaic.git"
@@ -25,15 +32,16 @@
     "to-parquet": "./bin/to-parquet.js"
   },
   "scripts": {
+    "clean": "rimraf dist && mkdir dist",
     "lint": "eslint src test",
     "server": "node bin/run-server.js",
     "test": "vitest run",
-    "prepublishOnly": "npm run test && npm run lint"
+    "prepublishOnly": "npm run test && npm run lint && tsc --build"
   },
   "dependencies": {
-    "@uwdata/mosaic-sql": "^0.16.2",
-    "duckdb": "^1.2.1",
+    "@uwdata/mosaic-sql": "^0.18.0",
+    "duckdb": "~1.2.1",
     "ws": "^8.18.2"
   },
-  "gitHead": "26d2719f4bcab471d2831145e1f03f39f3509869"
+  "gitHead": "dfb9ded0b0307754e3185ca34cc49a1384fe8455"
 }

package/src/Cache.js

@@ -93,7 +93,7 @@ async function readEntries(dir, cache) {
   let files;
   try {
     files = await fs.readdir(dir);
-  } catch (err) { // eslint-disable-line no-unused-vars
+  } catch (err) {
     return; // dir does not exist, nothing to do
   }
   await Promise.allSettled(files.map(async file => {

package/src/data-server.js

@@ -1,12 +1,9 @@
 import http from 'node:http';
-import path from 'node:path';
 import url from 'node:url';
 import { WebSocketServer } from 'ws';
 import { Cache, cacheKey } from './Cache.js';
-import { createBundle, loadBundle } from './load/bundle.js';
 
 const CACHE_DIR = '.mosaic/cache';
-const BUNDLE_DIR = '.mosaic/bundle';
 
 export function dataServer(db, {
   cache = true,
@@ -121,19 +118,6 @@ export function queryHandler(db, queryCache) {
           // JSON response format
           res.json(await retrieve(query, sql => db.query(sql)));
           break;
-        case 'create-bundle':
-          // Create a named bundle of precomputed resources
-          await createBundle(
-            db, queryCache, query.queries,
-            path.resolve(BUNDLE_DIR, query.name)
-          );
-          res.done();
-          break;
-        case 'load-bundle':
-          // Load a named bundle of precomputed resources
-          await loadBundle(db, queryCache, path.resolve(BUNDLE_DIR, query.name));
-          res.done();
-          break;
         default:
           res.error(`Unrecognized command: ${type}`, 400);
       }

package/src/load/bundle.js

@@ -1,75 +0,0 @@
-import fs from 'node:fs/promises';
-import path from 'node:path';
-import { cacheKey } from '../Cache.js';
-
-async function retrieve(db, cache, sql, type) {
-  const key = cacheKey(sql, type);
-  const cached = cache.get(key);
-  if (cached) return cached;
-  switch (type) {
-    case 'arrow':
-      return db.arrowBuffer(sql);
-    case 'json':
-      return JSON.stringify(await db.query(sql));
-    default:
-      throw new Error(`Unsupported query type: ${type}`);
-  }
-}
-
-export async function createBundle(db, cache, queries, dir) {
-  const describe_re = /^DESCRIBE /;
-  const pragma_re = /^PRAGMA /;
-  const view_re = /^CREATE( TEMP| TEMPORARY)? VIEW/;
-  const table_re = /^CREATE( TEMP| TEMPORARY)? TABLE( IF NOT EXISTS)? ([^\s]+)/;
-
-  const manifest = { tables: [], queries: [] };
-
-  await fs.mkdir(dir, { recursive: true });
-
-  const querySet = new Set(queries);
-  for (const query of querySet) {
-    const sql = typeof query === 'string' ? query : query.sql;
-    if (query.alias) {
-      const table = query.alias;
-      const file = path.resolve(dir, `${table}.parquet`);
-      await db.exec(`COPY (${sql}) TO '${file}' (FORMAT PARQUET)`);
-      manifest.tables.push(table);
-    } else if (sql.startsWith('CREATE ')) {
-      // table or view
-      if (view_re.test(sql)) continue; // ignore views
-      const table = sql.match(table_re)?.[3];
-      const file = path.resolve(dir, `${table}.parquet`);
-      await db.exec(`${sql}`);
-      await db.exec(`COPY ${table} TO '${file}' (FORMAT PARQUET)`);
-      manifest.tables.push(table);
-    } else if (!pragma_re.test(sql)) {
-      // select query
-      const type = describe_re.test(sql) ? 'json' : 'arrow';
-      const key = cacheKey(sql, type);
-      const result = await retrieve(db, cache, sql, type);
-      await fs.writeFile(path.resolve(dir, key), result);
-      manifest.queries.push(key);
-    }
-  }
-
-  await fs.writeFile(path.resolve(dir, 'bundle.json'), JSON.stringify(manifest, 0, 2));
-  return manifest;
-}
-
-export async function loadBundle(db, cache, dir) {
-  const manifest = JSON.parse(await fs.readFile(path.resolve(dir, 'bundle.json')));
-
-  // load precomputed query results into the cache
-  for (const key of manifest.queries) {
-    const file = path.resolve(dir, key);
-    const json = path.extname(file) === '.json';
-    const data = await fs.readFile(file);
-    cache.set(key, json ? JSON.parse(data) : data);
-  }
-
-  // load precomputed tables into the database
-  for (const table of manifest.tables) {
-    const file = path.resolve(dir, `${table}.parquet`);
-    await db.exec(`CREATE TABLE IF NOT EXISTS ${table} AS SELECT * FROM '${file}'`);
-  }
-}

package/vitest.config.ts

@@ -1,3 +0,0 @@
-import { defineConfig } from 'vite';
-
-export default defineConfig({});