@malloydata/db-duckdb 0.0.330 → 0.0.331

package/dist/duckdb_wasm_connection.d.ts

@@ -1,24 +1,6 @@
 import * as duckdb from '@duckdb/duckdb-wasm';
 import type { FetchSchemaOptions, QueryDataRow, QueryOptionsReader, RunSQLOptions, SQLSourceDef, ConnectionConfig, TableSourceDef, SQLSourceRequest } from '@malloydata/malloy';
-import type { StructRow } from 'apache-arrow';
 import { DuckDBCommon } from './duckdb_common';
-/**
- * Arrow's toJSON() doesn't really do what I'd expect, since
- * it still includes Arrow objects like DecimalBigNums and Vectors,
- * so we need this fairly gross function to unwrap those.
- *
- * @param value Element from an Arrow StructRow.
- * @return Vanilla Javascript value
- */
-export declare const unwrapArrow: (value: unknown) => any;
-/**
- * Process a single Arrow result row into a Malloy QueryDataRow
- * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
- * winds up converting DecimalBigNums to strings instead of numbers.
- * For some reason a custom replacer only sees DecimalBigNums as
- * strings, as well.
- */
-export declare const unwrapRow: (row: StructRow) => QueryDataRow;
 type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export interface DuckDBWasmOptions extends ConnectionConfig {
     additionalExtensions?: string[];

package/dist/duckdb_wasm_connection.js

@@ -58,105 +58,173 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.DuckDBWASMConnection = exports.unwrapRow = exports.unwrapArrow = void 0;
+exports.DuckDBWASMConnection = void 0;
 const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const web_worker_1 = __importDefault(require("web-worker"));
+const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 const TABLE_MATCH = /FROM\s*('([^']*)'|"([^"]*)")/gi;
 const TABLE_FUNCTION_MATCH = /FROM\s+[a-z0-9_]+\(('([^']*)'|"([^"]*)")/gi;
 const FILE_EXTS = ['.csv', '.tsv', '.parquet'];
-const isIterable = (x) => Symbol.iterator in x;
+// ----------------------------------------------------------------------------
+// Arrow value unwrapping functions
+// These convert Arrow values to vanilla JS using schema type information.
+// ----------------------------------------------------------------------------
 /**
- * Arrow's toJSON() doesn't really do what I'd expect, since
- * it still includes Arrow objects like DecimalBigNums and Vectors,
- * so we need this fairly gross function to unwrap those.
- *
- * @param value Element from an Arrow StructRow.
- * @return Vanilla Javascript value
+ * Convert an Arrow value to vanilla JS using the Arrow DataType.
+ * Uses schema type info to correctly handle decimals and nested types.
  */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
-const unwrapArrow = (value) => {
-    if (value === null) {
-        return value;
-    }
-    else if (value instanceof Date) {
-        return value;
-    }
-    else if (typeof value === 'bigint') {
-        // Safe bigints can be represented as numbers without precision loss
-        if (value >= BigInt(Number.MIN_SAFE_INTEGER) &&
-            value <= BigInt(Number.MAX_SAFE_INTEGER)) {
-            return Number(value);
-        }
-        // Large bigints stay as strings to preserve precision
-        return value.toString();
+function unwrapValue(value, fieldType) {
+    if (value === null || value === undefined) {
+        return null;
     }
-    else if (typeof value === 'object') {
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const obj = value;
-        // DecimalBigNums appear as Uint32Arrays, but can be identified
-        // because they have a Symbol.toPrimitive method
-        if (obj[Symbol.toPrimitive]) {
-            const primitiveValue = obj[Symbol.toPrimitive]();
-            if (typeof primitiveValue === 'string') {
-                const num = Number(primitiveValue);
-                // Safe integers can be represented as numbers without precision loss
-                if (Number.isSafeInteger(num)) {
-                    return num;
-                }
-                // Large numbers stay as strings to preserve precision (HUGEINT)
-                return primitiveValue;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const children = fieldType.children;
+    switch (fieldType.typeId) {
+        case apache_arrow_1.Type.Decimal:
+            return unwrapDecimal(value, fieldType);
+        case apache_arrow_1.Type.Date:
+        case apache_arrow_1.Type.Timestamp:
+            if (typeof value === 'number') {
+                return new Date(value);
             }
-            if (typeof primitiveValue === 'number') {
-                return primitiveValue;
+            if (value instanceof Date) {
+                return value;
             }
-            if (typeof primitiveValue === 'bigint') {
-                if (primitiveValue >= BigInt(Number.MIN_SAFE_INTEGER) &&
-                    primitiveValue <= BigInt(Number.MAX_SAFE_INTEGER)) {
-                    return Number(primitiveValue);
-                }
-                return primitiveValue.toString();
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.List:
+        case apache_arrow_1.Type.FixedSizeList:
+            if (children && children.length > 0) {
+                return unwrapArray(value, children[0].type);
             }
-            return primitiveValue.toString();
-        }
-        else if (Array.isArray(value)) {
-            return value.map(exports.unwrapArrow);
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.Struct:
+            if (children && children.length > 0) {
+                return unwrapStruct(value, children);
+            }
+            return unwrapPrimitive(value);
+        case apache_arrow_1.Type.Map:
+            // Maps have a single child which is a struct with key/value fields
+            if (children && children.length > 0) {
+                return unwrapArray(value, children[0].type);
+            }
+            return unwrapPrimitive(value);
+        default:
+            return unwrapPrimitive(value);
+    }
+}
+function unwrapDecimal(value, fieldType) {
+    var _a;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const scale = (_a = fieldType.scale) !== null && _a !== void 0 ? _a : 0;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    if (!obj || !obj[Symbol.toPrimitive]) {
+        return value;
+    }
+    const raw = obj[Symbol.toPrimitive]();
+    if (typeof raw === 'bigint') {
+        // Check if the unscaled value exceeds safe integer range
+        const absRaw = raw < BigInt(0) ? -raw : raw;
+        if (absRaw > BigInt(Number.MAX_SAFE_INTEGER)) {
+            // Too large for precise JS number - format as decimal string
+            return formatBigDecimal(raw, scale);
         }
-        else if (isIterable(value)) {
-            // Catch Arrow Vector objects
-            return [...value].map(exports.unwrapArrow);
+        if (scale > 0) {
+            return Number(raw) / 10 ** scale;
         }
-        else {
-            // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            const result = {};
-            for (const key in obj) {
-                if (Object.prototype.hasOwnProperty.call(obj, key)) {
-                    result[key] = (0, exports.unwrapArrow)(obj[key]);
-                }
-            }
-            return result;
+        return Number(raw);
+    }
+    if (typeof raw === 'string') {
+        // Large decimals may come as strings - check if too large for Number
+        const absStr = raw.startsWith('-') ? raw.slice(1) : raw;
+        if (absStr.length > 15) {
+            // String is likely too large for precise Number - format with decimal
+            return formatBigDecimalFromString(raw, scale);
         }
     }
+    const num = Number(raw);
+    return scale > 0 ? num / 10 ** scale : num;
+}
+function unwrapArray(value, elementType) {
+    const arr = Array.isArray(value) ? value : [...value];
+    return arr.map(v => unwrapValue(v, elementType));
+}
+function unwrapStruct(value, children) {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    const result = {};
+    for (const field of children) {
+        result[field.name] = unwrapValue(obj[field.name], field.type);
+    }
+    return result;
+}
+function unwrapPrimitive(value) {
+    if (value instanceof Date)
+        return value;
+    if (typeof value === 'bigint')
+        return safeNumber(value);
+    if (typeof value !== 'object' || value === null)
+        return value;
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const obj = value;
+    if (obj[Symbol.toPrimitive]) {
+        return safeNumber(obj[Symbol.toPrimitive]());
+    }
     return value;
-};
-exports.unwrapArrow = unwrapArrow;
+}
+function safeNumber(value) {
+    if (typeof value === 'number') {
+        return value;
+    }
+    const num = Number(value);
+    if (Number.isSafeInteger(num) ||
+        (Number.isFinite(num) && !Number.isInteger(num))) {
+        return num;
+    }
+    return String(value);
+}
+function formatBigDecimal(raw, scale) {
+    const isNegative = raw < BigInt(0);
+    const str = (isNegative ? -raw : raw).toString();
+    return formatDecimalString(str, scale, isNegative);
+}
+function formatBigDecimalFromString(raw, scale) {
+    const isNegative = raw.startsWith('-');
+    const str = isNegative ? raw.slice(1) : raw;
+    return formatDecimalString(str, scale, isNegative);
+}
+function formatDecimalString(str, scale, isNegative) {
+    let result;
+    if (scale <= 0) {
+        result = str;
+    }
+    else if (scale >= str.length) {
+        result = '0.' + '0'.repeat(scale - str.length) + str;
+    }
+    else {
+        result = str.slice(0, -scale) + '.' + str.slice(-scale);
+    }
+    return isNegative ? '-' + result : result;
+}
 /**
- * Process a single Arrow result row into a Malloy QueryDataRow
- * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
- * winds up converting DecimalBigNums to strings instead of numbers.
- * For some reason a custom replacer only sees DecimalBigNums as
- * strings, as well.
+ * Process a single Arrow result row into a Malloy QueryDataRow.
  */
-const unwrapRow = (row) => {
-    return (0, exports.unwrapArrow)(row.toJSON());
-};
-exports.unwrapRow = unwrapRow;
+function unwrapRow(row, schema) {
+    const json = row.toJSON();
+    const result = {};
+    for (const field of schema.fields) {
+        // Cast is safe: unwrapValue returns QueryValue-compatible types
+        result[field.name] = unwrapValue(json[field.name], field.type);
+    }
+    return result;
+}
 /**
- * Process a duckedb Table into an array of Malloy QueryDataRows
+ * Process a DuckDB Table into an array of Malloy QueryDataRows.
  */
-const unwrapTable = (table) => {
-    return table.toArray().map(exports.unwrapRow);
-};
+function unwrapTable(table) {
+    return table.toArray().map(row => unwrapRow(row, table.schema));
+}
 const isNode = () => typeof navigator === 'undefined';
 class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     constructor(arg, arg2, workingDirectory, queryOptions) {
@@ -329,7 +397,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
                     (abortSignal === null || abortSignal === void 0 ? void 0 : abortSignal.aborted)) {
                     break;
                 }
-                yield (0, exports.unwrapRow)(row);
+                yield unwrapRow(row, chunk.schema);
                 index++;
             }
         }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.330",
+  "version": "0.0.331",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -45,7 +45,7 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "1.29.1-dev132.0",
-    "@malloydata/malloy": "0.0.330",
+    "@malloydata/malloy": "0.0.331",
     "@motherduck/wasm-client": "^0.6.6",
     "apache-arrow": "^17.0.0",
     "duckdb": "1.3.4",