@sqlrooms/duckdb 0.0.0

package/dist/useDuckDb.js

@@ -0,0 +1,231 @@
+import * as duckdb from '@duckdb/duckdb-wasm';
+const ENABLE_DUCK_LOGGING = false;
+const SilentLogger = {
+    log: () => {
+        /* do nothing */
+    },
+};
+// TODO: shut DB down at some point
+let duckConn;
+let initialize;
+export class DuckQueryError extends Error {
+    cause;
+    query;
+    queryCallStack;
+    constructor(err, query, stack) {
+        super(`DB query failed: ${err instanceof Error ? err.message : err}\n\nFull query:\n\n${query}\n\nQuery call stack:\n\n${stack}\n\n`);
+        this.cause = err;
+        this.query = query;
+        this.queryCallStack = stack;
+        Object.setPrototypeOf(this, DuckQueryError.prototype);
+    }
+    getMessageForUser() {
+        const msg = this.cause instanceof Error ? this.cause.message : this.message;
+        return msg;
+    }
+}
+/**
+ * @deprecated getDuckConn is deprecated, use getDuckDb instead
+ */
+export const getDuckConn = getDuckDb;
+export async function getDuckDb() {
+    if (!globalThis.Worker) {
+        return Promise.reject('No Worker support');
+    }
+    if (duckConn) {
+        return duckConn;
+    }
+    else if (initialize !== undefined) {
+        // The initialization has already been started, wait for it to finish
+        return initialize;
+    }
+    let resolve;
+    let reject;
+    initialize = new Promise((_resolve, _reject) => {
+        resolve = _resolve;
+        reject = _reject;
+    });
+    try {
+        // TODO: Consider to load locally https://github.com/duckdb/duckdb-wasm/issues/1425#issuecomment-1742156605
+        const allBundles = duckdb.getJsDelivrBundles();
+        const bestBundle = await duckdb.selectBundle(allBundles);
+        if (!bestBundle.mainWorker) {
+            throw new Error('No best bundle found for DuckDB worker');
+        }
+        const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bestBundle.mainWorker}");`], {
+            type: 'text/javascript',
+        }));
+        // const worker = await duckdb.createWorker(bestBundle.mainWorker);
+        const worker = new window.Worker(workerUrl);
+        const logger = ENABLE_DUCK_LOGGING
+            ? new duckdb.ConsoleLogger()
+            : SilentLogger;
+        const db = new (class extends duckdb.AsyncDuckDB {
+            onError(event) {
+                super.onError(event);
+                console.error('onError', event);
+            }
+        })(logger, worker);
+        await db.instantiate(bestBundle.mainModule, bestBundle.pthreadWorker);
+        URL.revokeObjectURL(workerUrl);
+        await db.open({
+            path: ':memory:',
+            query: {
+            // castBigIntToDouble: true
+            },
+        });
+        const conn = await db.connect();
+        // Replace conn.query to include full query in the error message
+        const connQuery = conn.query;
+        conn.query = (async (q) => {
+            const stack = new Error().stack;
+            try {
+                return await connQuery.call(conn, q);
+            }
+            catch (err) {
+                throw new DuckQueryError(err, q, stack);
+                // throw new Error(
+                //   `Query failed: ${err}\n\nFull query:\n\n${q}\n\nQuery call stack:\n\n${stack}\n\n`,
+                // );
+            }
+        });
+        await conn.query(`
+      SET max_expression_depth TO 100000;
+      SET memory_limit = '10GB';
+    `);
+        duckConn = { db, conn, worker };
+        resolve(duckConn);
+    }
+    catch (err) {
+        reject(err);
+        throw err;
+    }
+    return duckConn;
+}
+// Cache the promise to avoid multiple initialization attempts
+let duckPromise = null;
+/**
+ * @deprecated useDuckConn is deprecated, use useDuckDb instead
+ */
+export const useDuckConn = useDuckDb;
+export function useDuckDb() {
+    if (!duckPromise) {
+        duckPromise = getDuckDb();
+    }
+    // If we don't have a connection yet, throw the promise
+    // This will trigger Suspense
+    if (!duckConn) {
+        throw duckPromise;
+    }
+    return duckConn;
+}
+export const isNumericDuckType = (type) => type.indexOf('INT') >= 0 ||
+    type.indexOf('DECIMAL') >= 0 ||
+    type.indexOf('FLOAT') >= 0 ||
+    type.indexOf('REAL') >= 0 ||
+    type.indexOf('DOUBLE') >= 0;
+export function getColValAsNumber(res, column = 0, index = 0) {
+    const v = (typeof column === 'number' ? res.getChildAt(column) : res.getChild(column))?.get(index);
+    if (v === undefined || v === null) {
+        return NaN;
+    }
+    // if it's an array (can be returned by duckdb as bigint)
+    return Number(v[0] ?? v);
+}
+export const escapeVal = (val) => {
+    return `'${String(val).replace(/'/g, "''")}'`;
+};
+export const escapeId = (id) => {
+    const str = String(id);
+    if (str.startsWith('"') && str.endsWith('"')) {
+        return str;
+    }
+    return `"${str.replace(/"/g, '""')}"`;
+};
+export async function getDuckTables(schema = 'main') {
+    const { conn } = await getDuckDb();
+    const tablesResults = await conn.query(`SELECT * FROM information_schema.tables 
+     WHERE table_schema = '${schema}'
+     ORDER BY table_name`);
+    const tableNames = [];
+    for (let i = 0; i < tablesResults.numRows; i++) {
+        tableNames.push(tablesResults.getChild('table_name')?.get(i));
+    }
+    return tableNames;
+}
+export async function getDuckTableSchema(tableName, schema = 'main') {
+    const { conn } = await getDuckDb();
+    const describeResults = await conn.query(`DESCRIBE ${schema}.${tableName}`);
+    const columnNames = describeResults.getChild('column_name');
+    const columnTypes = describeResults.getChild('column_type');
+    const columns = [];
+    for (let di = 0; di < describeResults.numRows; di++) {
+        const columnName = columnNames?.get(di);
+        const columnType = columnTypes?.get(di);
+        columns.push({ name: columnName, type: columnType });
+    }
+    return {
+        tableName,
+        columns,
+        // Costly to get the row count for large tables
+        // rowCount: getColValAsNumber(
+        //   await conn.query(`SELECT COUNT(*) FROM ${schema}.${tableName}`),
+        // ),
+    };
+}
+export async function getDuckTableSchemas(schema = 'main') {
+    const tableNames = await getDuckTables(schema);
+    const tablesInfo = [];
+    for (const tableName of tableNames) {
+        tablesInfo.push(await getDuckTableSchema(tableName, schema));
+    }
+    return tablesInfo;
+}
+export async function checkTableExists(tableName, schema = 'main') {
+    const { conn } = await getDuckDb();
+    const res = await conn.query(`SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = '${schema}' AND table_name = '${tableName}'`);
+    return getColValAsNumber(res) > 0;
+}
+export async function dropAllTables(schema) {
+    try {
+        const { conn } = await getDuckDb();
+        if (schema && schema !== 'main') {
+            await conn.query(`DROP SCHEMA IF EXISTS ${schema} CASCADE`);
+        }
+        else {
+            const res = await conn.query(`SELECT table_name, table_schema, table_type FROM information_schema.tables${schema ? ` WHERE table_schema = '${schema}'` : ''}`);
+            const schemasCol = res.getChild('table_schema');
+            const tableNamesCol = res.getChild('table_name');
+            const tableTypesCol = res.getChild('table_type');
+            for (let i = 0; i < res.numRows; i++) {
+                try {
+                    const schemaName = schemasCol?.get(i);
+                    const tableName = tableNamesCol?.get(i);
+                    const tableType = tableTypesCol?.get(i);
+                    if (tableName) {
+                        const query = `DROP ${tableType === 'VIEW' ? 'VIEW' : 'TABLE'} IF EXISTS ${schemaName}.${tableName}`;
+                        await conn.query(query);
+                    }
+                }
+                catch (err) {
+                    console.error(err);
+                }
+            }
+        }
+    }
+    catch (err) {
+        console.error(err);
+    }
+}
+export async function dropTable(tableName) {
+    const { conn } = await getDuckDb();
+    await conn.query(`DROP TABLE IF EXISTS ${tableName};`);
+}
+export async function dropFile(fname) {
+    const { db } = await getDuckDb();
+    db.dropFile(fname);
+}
+export async function dropAllFiles() {
+    const { db } = await getDuckDb();
+    db.dropFiles();
+}

package/eslint.config.js

@@ -0,0 +1,4 @@
+import {config} from '@sqlrooms/eslint-config/base';
+
+/** @type {import("eslint").Linter.Config} */
+export default config;

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@sqlrooms/duckdb",
+  "version": "0.0.0",
+  "main": "dist/index.js",
+  "types": "src/index.ts",
+  "module": "dist/index.js",
+  "type": "module",
+  "private": false,
+  "publishConfig": {
+    "access": "public"
+  },
+  "peerDependencies": {
+    "@duckdb/duckdb-wasm": "*",
+    "apache-arrow": "*"
+  },
+  "scripts": {
+    "dev": "tsc -w",
+    "build": "tsc",
+    "lint": "eslint ."
+  },
+  "gitHead": "4b0c709542475e4f95db0b2a8405ecadcf2ec186"
+}

package/src/duckdb.ts

@@ -0,0 +1,198 @@
+import {DuckDBDataProtocol} from '@duckdb/duckdb-wasm';
+import {escapeVal, getColValAsNumber, getDuckDb} from './useDuckDb';
+
+// export function makeTableName(inputFileName: string): string {
+//   return inputFileName.replace(/\.[^\.]*$/, '').replace(/\W/g, '_');
+// }
+
+export async function createTableFromQuery(tableName: string, query: string) {
+  const {conn} = await getDuckDb();
+  const rowCount = getColValAsNumber(
+    await conn.query(
+      `CREATE OR REPLACE TABLE main.${tableName} AS (
+        ${query}
+      )`,
+    ),
+  );
+  return {tableName, rowCount};
+}
+
+export async function createViewFromRegisteredFile(
+  filePath: string,
+  schema: string,
+  tableName: string,
+  opts?: {
+    // columnSpecs?: ColumnSpec[];
+    mode: 'table' | 'view';
+  },
+): Promise<{tableName: string; rowCount: number}> {
+  const {mode = 'table'} = opts ?? {};
+  const {conn} = await getDuckDb();
+  const fileNameLower = filePath.toLowerCase();
+  // let rowCount: number;
+  // if (fileNameLower.endsWith('.json')) {
+  //   await conn.insertJSONFromPath(filePath, {schema, name: tableName});
+  //   // TODO: for JSON we can use insertJSONFromPath https://github.com/duckdb/duckdb-wasm/issues/1262
+  //   // fileNameLower.endsWith('.json') || fileNameLower.endsWith('.ndjson')
+  //   // ? `read_json_auto(${escapeVal(fileName)})`
+  //   rowCount = getColValAsNumber(
+  //     await conn.query(`SELECT COUNT(*) FROM ${schema}.${tableName}`),
+  //   );
+  // } else {
+  const quotedFileName = escapeVal(filePath);
+  const readFileQuery =
+    fileNameLower.endsWith('.json') ||
+    fileNameLower.endsWith('.geojson') ||
+    fileNameLower.endsWith('.ndjson')
+      ? `read_json_auto(${quotedFileName}, maximum_object_size=104857600)` // 100MB
+      : fileNameLower.endsWith('.parquet')
+        ? `parquet_scan(${quotedFileName})`
+        : fileNameLower.endsWith('.csv') || fileNameLower.endsWith('.tsv')
+          ? `read_csv(${quotedFileName}, SAMPLE_SIZE=-1, AUTO_DETECT=TRUE)`
+          : quotedFileName;
+  // const readFileQuery = fileNameLower.endsWith('.csv')
+  //   ? `read_csv(${quotedFileName}, SAMPLE_SIZE=-1, AUTO_DETECT=TRUE)`
+  //   : quotedFileName;
+
+  // TODO: tableName generate
+  const rowCount = getColValAsNumber(
+    await conn.query(
+      `CREATE OR REPLACE ${mode} ${schema}.${tableName} AS
+          SELECT * FROM ${readFileQuery}`,
+    ),
+  );
+  // }
+  return {tableName, rowCount};
+}
+
+export async function createViewFromFile(
+  filePath: string,
+  schema: string,
+  tableName: string,
+  file: File | Uint8Array,
+): Promise<{tableName: string; rowCount: number}> {
+  const duckConn = await getDuckDb();
+
+  // const fileName = file.name;
+  // await duckConn.db.dropFile(fileName);
+  // await duckConn.db.registerFileHandle(
+  //   fileName,
+  //   file,
+  //   DuckDBDataProtocol.BROWSER_FILEREADER,
+  //   true,
+  // );
+
+  // const tableName = makeTableName(fileName);
+  // await duckConn.conn.query(`
+  //     CREATE OR REPLACE VIEW ${tableName} AS SELECT * FROM '${fileName}'
+  // `);
+
+  //const fileName = file.name;
+  await duckConn.db.dropFile(filePath);
+  if (file instanceof File) {
+    await duckConn.db.registerFileHandle(
+      filePath,
+      file,
+      DuckDBDataProtocol.BROWSER_FILEREADER,
+      true,
+    );
+  } else {
+    await duckConn.db.registerFileBuffer(filePath, file);
+  }
+
+  return createViewFromRegisteredFile(filePath, schema, tableName);
+
+  // const res = await duckConn.conn.query(
+  //   `SELECT count(*) FROM ${inputTableName}`,
+  // );
+  // const inputRowCount = getColValAsNumber(res, 0);
+  // const tableMeta = await duckConn.conn.query(
+  //   `DESCRIBE TABLE ${inputTableName}`,
+  // );
+  // const inputTableFields = Array.from(tableMeta).map((row) => ({
+  //   name: String(row?.column_name),
+  //   type: String(row?.column_type),
+  // }));
+
+  // const nextResult: DataTable = {
+  //   inputFileName,
+  //   tableName: inputTableName,
+  //   rowCount: inputRowCount,
+  //   // outputRowCount: undefined,
+  //   columns: inputTableFields,
+  // };
+  // // setResult(nextResult);
+  // return nextResult;
+}
+
+// async function createViewFromFile2(
+//   file: File,
+//   duckConn: DuckDb,
+//   onTableCreated: (
+//     inputTableName: string,
+//     result: CreateTableDropzoneResult,
+//   ) => void,
+//   onError: (status:'error', message: string) => void,
+// ) {
+//   try {
+//     const inputFileName = file.name;
+//     await duckConn.db.dropFile(inputFileName);
+//     await duckConn.db.registerFileHandle(
+//       inputFileName,
+//       file,
+//       DuckDBDataProtocol.BROWSER_FILEREADER,
+//       true,
+//     );
+
+//     const inputTableName = genRandomStr(10, inputFileName).toLowerCase();
+//     await duckConn.conn.query(`DROP TABLE IF EXISTS ${inputTableName}`);
+//     const readFileQuery = inputFileName.endsWith('.parquet')
+//       ? `parquet_scan(${escapeVal(inputFileName)})`
+//       : `read_csv(${escapeVal(
+//           inputFileName,
+//         )}, SAMPLE_SIZE=-1, AUTO_DETECT=TRUE)`;
+//     await duckConn.conn.query(
+//       `CREATE TABLE ${inputTableName} AS
+//             SELECT * FROM ${readFileQuery}`,
+//     );
+
+//     const res = await duckConn.conn.query(
+//       `SELECT count(*) FROM ${inputTableName}`,
+//     );
+//     const inputRowCount = getColValAsNumber(res, 0);
+//     const tableMeta = await duckConn.conn.query(
+//       `DESCRIBE TABLE ${inputTableName}`,
+//     );
+//     const inputTableFields = Array.from(tableMeta).map((row) => ({
+//       name: String(row?.column_name),
+//       type: String(row?.column_type),
+//     }));
+
+//     const nextResult: CreateTableDropzoneResult = {
+//       inputFileName,
+//       inputTableName,
+//       inputRowCount,
+//       // outputRowCount: undefined,
+//       inputTableFields,
+//       columns: {},
+//     };
+//     // setResult(nextResult);
+//     onTableCreated(inputTableName, nextResult);
+//   } catch (e) {
+//     console.error(e);
+//     onError(e instanceof Error ? e.message : String(e));
+//   }
+// }
+
+// async function maybeDropTable(
+//   value: CreateTableDropzoneResult,
+//   duckConn: DuckDb,
+// ) {
+//   const {inputFileName, inputTableName} = value || {};
+//   if (inputFileName) {
+//     await duckConn.db.dropFile(inputFileName);
+//   }
+//   if (inputTableName) {
+//     await duckConn.conn.query(`DROP TABLE IF EXISTS ${inputTableName};`);
+//   }
+// }

package/src/exportToCsv.ts

@@ -0,0 +1,101 @@
+import * as arrow from 'apache-arrow';
+import {getDuckDb} from './useDuckDb';
+
+export async function exportToCsv(
+  query: string,
+  fileName: string,
+  pageSize = 100000,
+) {
+  const {conn} = await getDuckDb();
+
+  let offset = 0;
+  const blobs: Blob[] = [];
+  let headersAdded = false;
+
+  while (true) {
+    const currentQuery = `(
+        ${query}
+    ) LIMIT ${pageSize} OFFSET ${offset}`;
+    const results = await conn.query(currentQuery);
+
+    // Check if we received any results; if not, we are done.
+    if (results.numRows === 0) {
+      break;
+    }
+
+    const csvChunk = convertToCsv(results, !headersAdded);
+    blobs.push(new Blob([csvChunk], {type: 'text/csv'}));
+
+    // Ensure that headers are not added in subsequent iterations
+    headersAdded = true;
+
+    // Increment offset to fetch the next chunk
+    offset += pageSize;
+  }
+
+  const fullCsvBlob = new Blob(blobs, {type: 'text/csv'});
+  downloadBlob(fullCsvBlob, fileName);
+}
+
+function convertToCsv(
+  arrowTable: arrow.Table,
+  includeHeaders: boolean,
+): string {
+  // return includeHeaders
+  //   ? csvFormat(arrowTable.toArray())
+  //   : csvFormatBody(arrowTable.toArray());
+
+  const columnNames = arrowTable.schema.fields.map((field) => field.name);
+  const columnsByName = columnNames.reduce(
+    (acc, columnName) => {
+      const col = arrowTable.getChild(columnName);
+      if (col) acc[columnName] = col;
+      return acc;
+    },
+    {} as Record<string, arrow.Vector>,
+  );
+
+  // Add header
+  let csvContent = includeHeaders ? columnNames.join(',') + '\r\n' : '';
+
+  // Add data rows
+  for (let i = 0; i < arrowTable.numRows; i++) {
+    const csvRow = columnNames
+      .map((columnName) => {
+        const cellValue = columnsByName[columnName]?.get(i);
+
+        // If the cell value is null or undefined, set it to an empty string.
+        if (cellValue == null) return '';
+
+        // Convert cell value to string
+        let cellValueStr = String(cellValue);
+
+        // Escape double quotes and wrap cell value in double quotes if necessary
+        if (
+          cellValueStr.includes('"') ||
+          cellValueStr.includes(',') ||
+          cellValueStr.includes('\n')
+        ) {
+          cellValueStr = '"' + cellValueStr.replace(/"/g, '""') + '"';
+        }
+
+        return cellValueStr;
+      })
+      .join(',');
+
+    csvContent += csvRow + '\r\n';
+  }
+
+  return csvContent;
+}
+
+function downloadBlob(blob: Blob, filename: string) {
+  const url = URL.createObjectURL(blob);
+  const a = document.createElement('a');
+  a.href = url;
+  a.download = filename;
+  document.body.appendChild(a);
+  a.click();
+  URL.revokeObjectURL(url);
+  document.body.removeChild(a);
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './duckdb';
+export * from './types';
+export * from './useDuckDb';
+export * from './exportToCsv';

package/src/types.ts

@@ -0,0 +1,11 @@
+export type TableColumn = {
+  name: string;
+  type: string;
+};
+
+export type DataTable = {
+  tableName: string;
+  columns: TableColumn[];
+  rowCount?: number;
+  inputFileName?: string;
+};