@malloydata/db-duckdb 0.0.330 → 0.0.332

package/dist/duckdb_connection.d.ts

@@ -1,6 +1,6 @@
 import { DuckDBCommon } from './duckdb_common';
-import type { Connection, TableData } from 'duckdb';
-import { Database } from 'duckdb';
+import { DuckDBInstance } from '@duckdb/node-api';
+import type { DuckDBConnection as DuckDBNodeConnection } from '@duckdb/node-api';
 import type { ConnectionConfig, QueryDataRow, QueryOptionsReader, RunSQLOptions } from '@malloydata/malloy';
 export interface DuckDBConnectionOptions extends ConnectionConfig {
     additionalExtensions?: string[];
@@ -10,8 +10,8 @@ export interface DuckDBConnectionOptions extends ConnectionConfig {
     readOnly?: boolean;
 }
 interface ActiveDB {
-    database: Database;
-    connections: Connection[];
+    instance: DuckDBInstance;
+    connections: DuckDBNodeConnection[];
 }
 export declare class DuckDBConnection extends DuckDBCommon {
     readonly name: string;
@@ -20,7 +20,7 @@ export declare class DuckDBConnection extends DuckDBCommon {
     private workingDirectory;
     private readOnly;
     connecting: Promise<void>;
-    protected connection: Connection | null;
+    protected connection: DuckDBNodeConnection | null;
     protected setupError: Error | undefined;
     protected isSetup: Promise<void> | undefined;
     static activeDBs: Record<string, ActiveDB>;
@@ -30,7 +30,7 @@ export declare class DuckDBConnection extends DuckDBCommon {
     loadExtension(ext: string): Promise<void>;
     protected setup(): Promise<void>;
     protected runDuckDBQuery(sql: string): Promise<{
-        rows: TableData;
+        rows: QueryDataRow[];
         totalRows: number;
     }>;
     runSQLStream(sql: string, { rowLimit, abortSignal }?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;

package/dist/duckdb_connection.js

@@ -28,7 +28,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBConnection = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const duckdb_common_1 = require("./duckdb_common");
-const duckdb_1 = require("duckdb");
+const node_api_1 = require("@duckdb/node-api");
 const package_json_1 = __importDefault(require("@malloydata/malloy/package.json"));
 class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
@@ -80,24 +80,22 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         this.connecting = this.init();
     }
     async init() {
-        return new Promise(resolve => {
+        try {
             if (this.databasePath in DuckDBConnection.activeDBs) {
                 const activeDB = DuckDBConnection.activeDBs[this.databasePath];
-                this.connection = activeDB.database.connect();
+                this.connection = await activeDB.instance.connect();
                 activeDB.connections.push(this.connection);
-                resolve();
             }
             else {
                 const config = {
-                    'custom_user_agent': `Malloy/${package_json_1.default.version}`,
+                    custom_user_agent: `Malloy/${package_json_1.default.version}`,
                 };
                 if (this.isMotherDuck) {
                     if (!this.motherDuckToken &&
                         !process.env['motherduck_token'] &&
                         !process.env['MOTHERDUCK_TOKEN']) {
                         this.setupError = new Error('Please set your MotherDuck Token');
-                        // Resolve instead of error because errors cannot be caught.
-                        return resolve();
+                        return;
                     }
                     if (this.motherDuckToken) {
                         config['motherduck_token'] = this.motherDuckToken;
@@ -106,23 +104,18 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
                 if (this.readOnly) {
                     config['access_mode'] = 'READ_ONLY';
                 }
-                const database = new duckdb_1.Database(this.databasePath, config, err => {
-                    if (err) {
-                        this.setupError = err;
-                    }
-                    else {
-                        this.connection = database.connect();
-                        const activeDB = {
-                            database,
-                            connections: [],
-                        };
-                        DuckDBConnection.activeDBs[this.databasePath] = activeDB;
-                        activeDB.connections.push(this.connection);
-                    }
-                    resolve();
-                });
+                const instance = await node_api_1.DuckDBInstance.create(this.databasePath, config);
+                this.connection = await instance.connect();
+                const activeDB = {
+                    instance,
+                    connections: [this.connection],
+                };
+                DuckDBConnection.activeDBs[this.databasePath] = activeDB;
             }
-        });
+        }
+        catch (err) {
+            this.setupError = err instanceof Error ? err : new Error(String(err));
+        }
     }
     async loadExtension(ext) {
         try {
@@ -172,25 +165,16 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         await this.isSetup;
     }
     async runDuckDBQuery(sql) {
-        return new Promise((resolve, reject) => {
-            if (this.connection) {
-                this.connection.all(sql, (err, rows) => {
-                    if (err) {
-                        reject(err);
-                    }
-                    else {
-                        // rows = processBigInts(rows);
-                        resolve({
-                            rows,
-                            totalRows: rows.length,
-                        });
-                    }
-                });
-            }
-            else {
-                reject(new Error('Connection not open'));
-            }
-        });
+        if (!this.connection) {
+            throw new Error('Connection not open');
+        }
+        const result = await this.connection.run(sql);
+        // getRowObjectsJson() converts nested types (LIST, STRUCT) to JS arrays/objects
+        const rows = (await result.getRowObjectsJson());
+        return {
+            rows,
+            totalRows: rows.length,
+        };
     }
     async *runSQLStream(sql, { rowLimit, abortSignal } = {}) {
         const defaultOptions = this.readQueryOptions();
@@ -204,14 +188,17 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
             await this.runDuckDBQuery(statements[0]);
             statements.shift();
         }
+        const result = await this.connection.stream(statements[0]);
         let index = 0;
-        for await (const row of this.connection.stream(statements[0])) {
-            if ((rowLimit !== undefined && index >= rowLimit) ||
-                (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
-                break;
+        for await (const chunk of result.yieldRowObjectJson()) {
+            for (const row of chunk) {
+                if ((rowLimit !== undefined && index >= rowLimit) ||
+                    (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
+                    return;
+                }
+                index++;
+                yield row;
             }
-            index++;
-            yield row;
         }
     }
     async createHash(sqlCommand) {
@@ -222,7 +209,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         if (activeDB) {
             activeDB.connections = activeDB.connections.filter(connection => connection !== this.connection);
             if (activeDB.connections.length === 0) {
-                activeDB.database.close();
+                activeDB.instance.closeSync();
                 delete DuckDBConnection.activeDBs[this.databasePath];
             }
         }

package/dist/duckdb_wasm_connection.d.ts

@@ -1,24 +1,6 @@
 import * as duckdb from '@duckdb/duckdb-wasm';
 import type { FetchSchemaOptions, QueryDataRow, QueryOptionsReader, RunSQLOptions, SQLSourceDef, ConnectionConfig, TableSourceDef, SQLSourceRequest } from '@malloydata/malloy';
-import type { StructRow } from 'apache-arrow';
 import { DuckDBCommon } from './duckdb_common';
-/**
- * Arrow's toJSON() doesn't really do what I'd expect, since
- * it still includes Arrow objects like DecimalBigNums and Vectors,
- * so we need this fairly gross function to unwrap those.
- *
- * @param value Element from an Arrow StructRow.
- * @return Vanilla Javascript value
- */
-export declare const unwrapArrow: (value: unknown) => any;
-/**
- * Process a single Arrow result row into a Malloy QueryDataRow
- * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
- * winds up converting DecimalBigNums to strings instead of numbers.
- * For some reason a custom replacer only sees DecimalBigNums as
- * strings, as well.
- */
-export declare const unwrapRow: (row: StructRow) => QueryDataRow;
 type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export interface DuckDBWasmOptions extends ConnectionConfig {
     additionalExtensions?: string[];

package/dist/duckdb_wasm_connection.js

@@ -58,105 +58,173 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DuckDBWASMConnection = exports.unwrapRow = exports.unwrapArrow = void 0;
+exports.DuckDBWASMConnection = void 0;
 const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const web_worker_1 = __importDefault(require("web-worker"));
+const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 const TABLE_MATCH = /FROM\s*('([^']*)'|"([^"]*)")/gi;
 const TABLE_FUNCTION_MATCH = /FROM\s+[a-z0-9_]+\(('([^']*)'|"([^"]*)")/gi;
 const FILE_EXTS = ['.csv', '.tsv', '.parquet'];
-const isIterable = (x) => Symbol.iterator in x;
+// ----------------------------------------------------------------------------
+// Arrow value unwrapping functions
+// These convert Arrow values to vanilla JS using schema type information.
+// ----------------------------------------------------------------------------
 /**
- * Arrow's toJSON() doesn't really do what I'd expect, since
- * it still includes Arrow objects like DecimalBigNums and Vectors,
- * so we need this fairly gross function to unwrap those.
- *
- * @param value Element from an Arrow StructRow.
- * @return Vanilla Javascript value
+ * Convert an Arrow value to vanilla JS using the Arrow DataType.
+ * Uses schema type info to correctly handle decimals and nested types.
  */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const unwrapArrow = (value) => {
-    if (value === null) {
-        return value;
-    }
-    else if (value instanceof Date) {
-        return value;
-    }
-    else if (typeof value === 'bigint') {
-        // Safe bigints can be represented as numbers without precision loss
-        if (value >= BigInt(Number.MIN_SAFE_INTEGER) &&
-            value <= BigInt(Number.MAX_SAFE_INTEGER)) {
-            return Number(value);
-        }
-        // Large bigints stay as strings to preserve precision
-        return value.toString();
+function unwrapValue(value, fieldType) {
+    if (value === null || value === undefined) {
+        return null;
     }
-    else if (typeof value === 'object') {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const obj = value;
-        // DecimalBigNums appear as Uint32Arrays, but can be identified
-        // because they have a Symbol.toPrimitive method
-        if (obj[Symbol.toPrimitive]) {
-            const primitiveValue = obj[Symbol.toPrimitive]();
-            if (typeof primitiveValue === 'string') {
-                const num = Number(primitiveValue);
-                // Safe integers can be represented as numbers without precision loss
-                if (Number.isSafeInteger(num)) {
-                    return num;
-                }
-                // Large numbers stay as strings to preserve precision (HUGEINT)
-                return primitiveValue;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const children = fieldType.children;
+    switch (fieldType.typeId) {
+        case apache_arrow_1.Type.Decimal:
+            return unwrapDecimal(value, fieldType);
+        case apache_arrow_1.Type.Date:
+        case apache_arrow_1.Type.Timestamp:
+            if (typeof value === 'number') {
+                return new Date(value);
             }
-            if (typeof primitiveValue === 'number') {
-                return primitiveValue;
+            if (value instanceof Date) {
+                return value;
             }
-            if (typeof primitiveValue === 'bigint') {
-                if (primitiveValue >= BigInt(Number.MIN_SAFE_INTEGER) &&
-                    primitiveValue <= BigInt(Number.MAX_SAFE_INTEGER)) {
-                    return Number(primitiveValue);
-                }
-                return primitiveValue.toString();
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.List:
+        case apache_arrow_1.Type.FixedSizeList:
+            if (children && children.length > 0) {
+                return unwrapArray(value, children[0].type);
             }
-            return primitiveValue.toString();
-        }
-        else if (Array.isArray(value)) {
-            return value.map(exports.unwrapArrow);
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.Struct:
+            if (children && children.length > 0) {
+                return unwrapStruct(value, children);
+            }
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.Map:
+            // Maps have a single child which is a struct with key/value fields
+            if (children && children.length > 0) {
+                return unwrapArray(value, children[0].type);
+            }
+            return unwrapPrimitive(value);
+        default:
+            return unwrapPrimitive(value);
+    }
+}
+function unwrapDecimal(value, fieldType) {
+    var _a;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const scale = (_a = fieldType.scale) !== null && _a !== void 0 ? _a : 0;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    if (!obj || !obj[Symbol.toPrimitive]) {
+        return value;
+    }
+    const raw = obj[Symbol.toPrimitive]();
+    if (typeof raw === 'bigint') {
+        // Check if the unscaled value exceeds safe integer range
+        const absRaw = raw < BigInt(0) ? -raw : raw;
+        if (absRaw > BigInt(Number.MAX_SAFE_INTEGER)) {
+            // Too large for precise JS number - format as decimal string
+            return formatBigDecimal(raw, scale);
         }
-        else if (isIterable(value)) {
-            // Catch Arrow Vector objects
-            return [...value].map(exports.unwrapArrow);
+        if (scale > 0) {
+            return Number(raw) / 10 ** scale;
         }
-        else {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const result = {};
-            for (const key in obj) {
-                if (Object.prototype.hasOwnProperty.call(obj, key)) {
-                    result[key] = (0, exports.unwrapArrow)(obj[key]);
-                }
-            }
-            return result;
+        return Number(raw);
+    }
+    if (typeof raw === 'string') {
+        // Large decimals may come as strings - check if too large for Number
+        const absStr = raw.startsWith('-') ? raw.slice(1) : raw;
+        if (absStr.length > 15) {
+            // String is likely too large for precise Number - format with decimal
+            return formatBigDecimalFromString(raw, scale);
         }
     }
+    const num = Number(raw);
+    return scale > 0 ? num / 10 ** scale : num;
+}
+function unwrapArray(value, elementType) {
+    const arr = Array.isArray(value) ? value : [...value];
+    return arr.map(v => unwrapValue(v, elementType));
+}
+function unwrapStruct(value, children) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    const result = {};
+    for (const field of children) {
+        result[field.name] = unwrapValue(obj[field.name], field.type);
+    }
+    return result;
+}
+function unwrapPrimitive(value) {
+    if (value instanceof Date)
+        return value;
+    if (typeof value === 'bigint')
+        return safeNumber(value);
+    if (typeof value !== 'object' || value === null)
+        return value;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    if (obj[Symbol.toPrimitive]) {
+        return safeNumber(obj[Symbol.toPrimitive]());
+    }
     return value;
-};
-exports.unwrapArrow = unwrapArrow;
+}
+function safeNumber(value) {
+    if (typeof value === 'number') {
+        return value;
+    }
+    const num = Number(value);
+    if (Number.isSafeInteger(num) ||
+        (Number.isFinite(num) && !Number.isInteger(num))) {
+        return num;
+    }
+    return String(value);
+}
+function formatBigDecimal(raw, scale) {
+    const isNegative = raw < BigInt(0);
+    const str = (isNegative ? -raw : raw).toString();
+    return formatDecimalString(str, scale, isNegative);
+}
+function formatBigDecimalFromString(raw, scale) {
+    const isNegative = raw.startsWith('-');
+    const str = isNegative ? raw.slice(1) : raw;
+    return formatDecimalString(str, scale, isNegative);
+}
+function formatDecimalString(str, scale, isNegative) {
+    let result;
+    if (scale <= 0) {
+        result = str;
+    }
+    else if (scale >= str.length) {
+        result = '0.' + '0'.repeat(scale - str.length) + str;
+    }
+    else {
+        result = str.slice(0, -scale) + '.' + str.slice(-scale);
+    }
+    return isNegative ? '-' + result : result;
+}
 /**
- * Process a single Arrow result row into a Malloy QueryDataRow
- * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
- * winds up converting DecimalBigNums to strings instead of numbers.
- * For some reason a custom replacer only sees DecimalBigNums as
- * strings, as well.
+ * Process a single Arrow result row into a Malloy QueryDataRow.
  */
-const unwrapRow = (row) => {
-    return (0, exports.unwrapArrow)(row.toJSON());
-};
-exports.unwrapRow = unwrapRow;
+function unwrapRow(row, schema) {
+    const json = row.toJSON();
+    const result = {};
+    for (const field of schema.fields) {
+        // Cast is safe: unwrapValue returns QueryValue-compatible types
+        result[field.name] = unwrapValue(json[field.name], field.type);
+    }
+    return result;
+}
 /**
- * Process a duckedb Table into an array of Malloy QueryDataRows
+ * Process a DuckDB Table into an array of Malloy QueryDataRows.
  */
-const unwrapTable = (table) => {
-    return table.toArray().map(exports.unwrapRow);
-};
+function unwrapTable(table) {
+    return table.toArray().map(row => unwrapRow(row, table.schema));
+}
 const isNode = () => typeof navigator === 'undefined';
 class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
@@ -224,6 +292,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             if (this.databasePath) {
                 await this._database.open({
                     path: this.databasePath,
+                    accessMode: duckdb.DuckDBAccessMode.AUTOMATIC,
                 });
             }
             URL.revokeObjectURL(workerUrl);
@@ -329,7 +398,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
                     (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
                     break;
                 }
-                yield (0, exports.unwrapRow)(row);
+                yield unwrapRow(row, chunk.schema);
                 index++;
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.330",
+  "version": "0.0.332",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -44,11 +44,11 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "1.29.1-dev132.0",
-    "@malloydata/malloy": "0.0.330",
+    "@duckdb/duckdb-wasm": "1.33.1-dev13.0",
+    "@malloydata/malloy": "0.0.332",
     "@motherduck/wasm-client": "^0.6.6",
     "apache-arrow": "^17.0.0",
-    "duckdb": "1.3.4",
+    "@duckdb/node-api": "1.4.3-r.1",
     "web-worker": "^1.3.0"
   }
 }