@malloydata/db-duckdb 0.0.24-dev230224214106 → 0.0.24-dev230301165401

package/dist/duckdb_common.d.ts

@@ -1,4 +1,4 @@
-import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from "@malloydata/malloy";
+import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from '@malloydata/malloy';
 export interface DuckDBQueryOptions {
     rowLimit: number;
 }

package/dist/duckdb_common.js

@@ -25,25 +25,25 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBCommon = void 0;
 const malloy_1 = require("@malloydata/malloy");
 const duckDBToMalloyTypes = {
-    "BIGINT": "number",
-    "DOUBLE": "number",
-    "VARCHAR": "string",
-    "DATE": "date",
-    "TIMESTAMP": "timestamp",
-    "TIME": "string",
-    "DECIMAL": "number",
-    "BOOLEAN": "boolean",
-    "INTEGER": "number"
+    BIGINT: 'number',
+    DOUBLE: 'number',
+    VARCHAR: 'string',
+    DATE: 'date',
+    TIMESTAMP: 'timestamp',
+    TIME: 'string',
+    DECIMAL: 'number',
+    BOOLEAN: 'boolean',
+    INTEGER: 'number',
 };
 class DuckDBCommon {
     constructor(queryOptions) {
         this.queryOptions = queryOptions;
         this.schemaCache = new Map();
         this.sqlSchemaCache = new Map();
-        this.name = "duckdb_common";
+        this.name = 'duckdb_common';
     }
     get dialectName() {
-        return "duckdb";
+        return 'duckdb';
     }
     readQueryOptions() {
         const options = DuckDBCommon.DEFAULT_QUERY_OPTIONS;
@@ -73,7 +73,7 @@ class DuckDBCommon {
         var _a;
         const defaultOptions = this.readQueryOptions();
         const rowLimit = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : defaultOptions.rowLimit;
-        const statements = sql.split("-- hack: split on this");
+        const statements = sql.split('-- hack: split on this');
         while (statements.length > 1) {
             await this.runRawSQL(statements[0]);
             statements.shift();
@@ -83,23 +83,23 @@ class DuckDBCommon {
         if (result.length > rowLimit) {
             result = result.slice(0, rowLimit);
         }
-        return { "rows": result, "totalRows": result.length };
+        return { rows: result, totalRows: result.length };
     }
     async getSQLBlockSchema(sqlRef) {
         const structDef = {
-            "type": "struct",
-            "dialect": "duckdb",
-            "name": sqlRef.name,
-            "structSource": {
-                "type": "sql",
-                "method": "subquery",
-                "sqlBlock": sqlRef
+            type: 'struct',
+            dialect: 'duckdb',
+            name: sqlRef.name,
+            structSource: {
+                type: 'sql',
+                method: 'subquery',
+                sqlBlock: sqlRef,
             },
-            "structRelationship": {
-                "type": "basetable",
-                "connectionName": this.name
+            structRelationship: {
+                type: 'basetable',
+                connectionName: this.name,
             },
-            "fields": []
+            fields: [],
         };
         await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.selectStr})`, structDef);
         return structDef;
@@ -117,27 +117,27 @@ class DuckDBCommon {
     splitColumns(s) {
         const columns = [];
         let parens = 0;
-        let column = "";
+        let column = '';
         let eatSpaces = true;
         for (let idx = 0; idx < s.length; idx++) {
             const c = s.charAt(idx);
-            if (eatSpaces && c === " ") {
+            if (eatSpaces && c === ' ') {
                 // Eat space
             }
             else {
                 eatSpaces = false;
-                if (!parens && c === ",") {
+                if (!parens && c === ',') {
                     columns.push(column);
-                    column = "";
+                    column = '';
                     eatSpaces = true;
                 }
                 else {
                     column += c;
                 }
-                if (c === "(") {
+                if (c === '(') {
                     parens += 1;
                 }
-                else if (c === ")") {
+                else if (c === ')') {
                     parens -= 1;
                 }
             }
@@ -152,7 +152,7 @@ class DuckDBCommon {
             //const [name, type] = c.split(" ", 1);
             const columnMatch = c.match(/^(?<name>[^\s]+) (?<type>.*)$/);
             if (columnMatch && columnMatch.groups) {
-                ret[columnMatch.groups["name"]] = columnMatch.groups["type"];
+                ret[columnMatch.groups['name']] = columnMatch.groups['type'];
             }
             else {
                 throw new Error(`Badly form Structure definition ${s}`);
@@ -164,26 +164,26 @@ class DuckDBCommon {
         for (const name in typeMap) {
             let duckDBType = typeMap[name];
             // Remove DECIMAL(x,y) precision to simplify lookup
-            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, "DECIMAL");
+            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, 'DECIMAL');
             let malloyType = duckDBToMalloyTypes[duckDBType];
             const arrayMatch = duckDBType.match(/(?<duckDBType>.*)\[\]$/);
             if (arrayMatch && arrayMatch.groups) {
-                duckDBType = arrayMatch.groups["duckDBType"];
+                duckDBType = arrayMatch.groups['duckDBType'];
             }
             const structMatch = duckDBType.match(/^STRUCT\((?<fields>.*)\)$/);
             if (structMatch && structMatch.groups) {
-                const newTypeMap = this.stringToTypeMap(structMatch.groups["fields"]);
+                const newTypeMap = this.stringToTypeMap(structMatch.groups['fields']);
                 const innerStructDef = {
-                    "type": "struct",
+                    type: 'struct',
                     name,
-                    "dialect": this.dialectName,
-                    "structSource": { "type": arrayMatch ? "nested" : "inline" },
-                    "structRelationship": {
-                        "type": arrayMatch ? "nested" : "inline",
-                        "field": name,
-                        "isArray": false
+                    dialect: this.dialectName,
+                    structSource: { type: arrayMatch ? 'nested' : 'inline' },
+                    structRelationship: {
+                        type: arrayMatch ? 'nested' : 'inline',
+                        field: name,
+                        isArray: false,
                     },
-                    "fields": []
+                    fields: [],
                 };
                 this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
                 structDef.fields.push(innerStructDef);
@@ -192,28 +192,28 @@ class DuckDBCommon {
                 if (arrayMatch) {
                     malloyType = duckDBToMalloyTypes[duckDBType];
                     const innerStructDef = {
-                        "type": "struct",
+                        type: 'struct',
                         name,
-                        "dialect": this.dialectName,
-                        "structSource": { "type": "nested" },
-                        "structRelationship": {
-                            "type": "nested",
-                            "field": name,
-                            "isArray": true
+                        dialect: this.dialectName,
+                        structSource: { type: 'nested' },
+                        structRelationship: {
+                            type: 'nested',
+                            field: name,
+                            isArray: true,
                         },
-                        "fields": [{ "type": malloyType, "name": "value" }]
+                        fields: [{ type: malloyType, name: 'value' }],
                     };
                     structDef.fields.push(innerStructDef);
                 }
                 else {
                     if (malloyType) {
-                        structDef.fields.push({ "type": malloyType, name });
+                        structDef.fields.push({ type: malloyType, name });
                     }
                     else {
                         structDef.fields.push({
-                            "type": "unsupported",
-                            "rawType": duckDBType.toLowerCase(),
-                            name
+                            type: 'unsupported',
+                            rawType: duckDBType.toLowerCase(),
+                            name,
                         });
                     }
                 }
@@ -224,7 +224,7 @@ class DuckDBCommon {
         const typeMap = {};
         const result = await this.runRawSQL(infoQuery);
         for (const row of result.rows) {
-            typeMap[row["column_name"]] = row["column_type"];
+            typeMap[row['column_name']] = row['column_type'];
         }
         this.fillStructDefFromTypeMap(structDef, typeMap);
     }
@@ -234,11 +234,11 @@ class DuckDBCommon {
         if (!inCache) {
             try {
                 inCache = {
-                    "structDef": await this.getSQLBlockSchema(sqlRef)
+                    structDef: await this.getSQLBlockSchema(sqlRef),
                 };
             }
             catch (error) {
-                inCache = { "error": error.message };
+                inCache = { error: error.message };
             }
             this.sqlSchemaCache.set(key, inCache);
         }
@@ -252,12 +252,12 @@ class DuckDBCommon {
             if (!inCache) {
                 try {
                     inCache = {
-                        "schema": await this.getTableSchema(tableURL)
+                        schema: await this.getTableSchema(tableURL),
                     };
                     this.schemaCache.set(tableURL, inCache);
                 }
                 catch (error) {
-                    inCache = { "error": error.message };
+                    inCache = { error: error.message };
                 }
             }
             if (inCache.schema !== undefined) {
@@ -272,15 +272,15 @@ class DuckDBCommon {
     async getTableSchema(tableURL) {
         const { tablePath } = (0, malloy_1.parseTableURI)(tableURL);
         const structDef = {
-            "type": "struct",
-            "name": tablePath,
-            "dialect": "duckdb",
-            "structSource": { "type": "table", tablePath },
-            "structRelationship": {
-                "type": "basetable",
-                "connectionName": this.name
+            type: 'struct',
+            name: tablePath,
+            dialect: 'duckdb',
+            structSource: { type: 'table', tablePath },
+            structRelationship: {
+                type: 'basetable',
+                connectionName: this.name,
             },
-            "fields": []
+            fields: [],
         };
         const quotedTablePath = tablePath.match(/[:*/]/)
             ? `'${tablePath}'`
@@ -293,7 +293,7 @@ class DuckDBCommon {
         return true;
     }
     async test() {
-        await this.runRawSQL("SELECT 1");
+        await this.runRawSQL('SELECT 1');
     }
     async manifestTemporaryTable(sqlCommand) {
         const hash = await this.createHash(sqlCommand);
@@ -306,6 +306,6 @@ class DuckDBCommon {
 }
 exports.DuckDBCommon = DuckDBCommon;
 DuckDBCommon.DEFAULT_QUERY_OPTIONS = {
-    "rowLimit": 10
+    rowLimit: 10,
 };
 //# sourceMappingURL=duckdb_common.js.map

package/dist/duckdb_connection.d.ts

@@ -1,6 +1,6 @@
-import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
-import { Connection, Database, Row } from "duckdb";
-import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
+import { DuckDBCommon, QueryOptionsReader } from './duckdb_common';
+import { Connection, Database, Row } from 'duckdb';
+import { QueryDataRow, RunSQLOptions } from '@malloydata/malloy';
 export declare class DuckDBConnection extends DuckDBCommon {
     readonly name: string;
     private databasePath;

package/dist/duckdb_connection.js

@@ -30,7 +30,7 @@ const crypto_1 = __importDefault(require("crypto"));
 const duckdb_common_1 = require("./duckdb_common");
 const duckdb_1 = require("duckdb");
 class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
-    constructor(name, databasePath = ":memory:", workingDirectory = ".", queryOptions) {
+    constructor(name, databasePath = ':memory:', workingDirectory = '.', queryOptions) {
         super(queryOptions);
         this.name = name;
         this.databasePath = databasePath;
@@ -42,7 +42,8 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     async init() {
         return new Promise((resolve, reject) => {
             this.database = new duckdb_1.Database(this.databasePath, duckdb_1.OPEN_READWRITE, // databasePath === ":memory:" ? OPEN_READWRITE : OPEN_READONLY,
-            (err) => {
+            // databasePath === ":memory:" ? OPEN_READWRITE : OPEN_READONLY,
+            err => {
                 if (err) {
                     reject(err);
                 }
@@ -62,7 +63,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
             }
             catch (error) {
                 // eslint-disable-next-line no-console
-                console.error("Unable to load json extension", error);
+                console.error('Unable to load json extension', error);
             }
             try {
                 await this.runDuckDBQuery("INSTALL 'httpfs'");
@@ -70,7 +71,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
             }
             catch (error) {
                 // eslint-disable-next-line no-console
-                console.error("Unable to load httpfs extension", error);
+                console.error('Unable to load httpfs extension', error);
             }
         };
         await this.connecting;
@@ -89,22 +90,22 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
                     else {
                         resolve({
                             rows,
-                            "totalRows": rows.length
+                            totalRows: rows.length,
                         });
                     }
                 });
             }
             else {
-                reject(new Error("Connection not open"));
+                reject(new Error('Connection not open'));
             }
         });
     }
     async *runSQLStream(sql, _options = {}) {
         await this.setup();
         if (!this.connection) {
-            throw new Error("Connection not open");
+            throw new Error('Connection not open');
         }
-        const statements = sql.split("-- hack: split on this");
+        const statements = sql.split('-- hack: split on this');
         while (statements.length > 1) {
             await this.runDuckDBQuery(statements[0]);
             statements.shift();
@@ -114,7 +115,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
         }
     }
     async createHash(sqlCommand) {
-        return crypto_1.default.createHash("md5").update(sqlCommand).digest("hex");
+        return crypto_1.default.createHash('md5').update(sqlCommand).digest('hex');
     }
     async close() {
         if (this.connection) {

package/dist/duckdb_wasm_connection.d.ts

@@ -1,6 +1,6 @@
-import * as duckdb from "@duckdb/duckdb-wasm";
-import { QueryDataRow, RunSQLOptions, StructDef } from "@malloydata/malloy";
-import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
+import * as duckdb from '@duckdb/duckdb-wasm';
+import { QueryDataRow, RunSQLOptions, StructDef } from '@malloydata/malloy';
+import { DuckDBCommon, QueryOptionsReader } from './duckdb_common';
 declare type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;

package/dist/duckdb_wasm_connection.js

@@ -73,10 +73,10 @@ const unwrapArrow = (value) => {
     else if (value instanceof Date) {
         return value;
     }
-    else if (typeof value === "bigint") {
+    else if (typeof value === 'bigint') {
         return Number(value);
     }
-    else if (typeof value === "object") {
+    else if (typeof value === 'object') {
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const obj = value;
         // DecimalBigNums appear as Uint32Arrays, but can be identified
@@ -116,9 +116,9 @@ const unwrapRow = (row) => {
 const unwrapTable = (table) => {
     return table.toArray().map(unwrapRow);
 };
-const isNode = () => typeof navigator === "undefined";
+const isNode = () => typeof navigator === 'undefined';
 class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
-    constructor(name, databasePath = null, workingDirectory = "/", queryOptions) {
+    constructor(name, databasePath = null, workingDirectory = '/', queryOptions) {
         super(queryOptions);
         this.name = name;
         this.databasePath = databasePath;
@@ -138,7 +138,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             const workerUrl = isNode()
                 ? bundle.mainWorker
                 : URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], {
-                    "type": "text/javascript"
+                    type: 'text/javascript',
                 }));
             // Instantiate the asynchronous version of DuckDB-wasm
             this.worker = new web_worker_1.default(workerUrl);
@@ -147,14 +147,14 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             await this._database.instantiate(bundle.mainModule, bundle.pthreadWorker);
             if (this.databasePath) {
                 await this._database.open({
-                    "path": this.databasePath
+                    path: this.databasePath,
                 });
             }
             URL.revokeObjectURL(workerUrl);
             this._connection = await this._database.connect();
         }
         else {
-            throw new Error("Unable to instantiate duckdb-wasm");
+            throw new Error('Unable to instantiate duckdb-wasm');
         }
     }
     get connection() {
@@ -169,25 +169,28 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     async runDuckDBQuery(sql) {
         var _a;
         const table = await ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.query(sql));
-        if ((table === null || table === void 0 ? void 0 : table.numRows) != null) {
+        if (table === undefined) {
+            throw new Error('Table is undefined.');
+        }
+        if ((table === null || table === void 0 ? void 0 : table.numRows) !== null) {
             const rows = unwrapTable(table);
             // console.log(rows);
             return {
                 // Normalize the data from its default proxied form
                 rows,
-                "totalRows": table.numRows
+                totalRows: table.numRows,
             };
         }
         else {
-            throw new Error("Boom");
+            throw new Error('Boom');
         }
     }
     async *runSQLStream(sql, _options = {}) {
         if (!this.connection) {
-            throw new Error("duckdb-wasm not connected");
+            throw new Error('duckdb-wasm not connected');
         }
         await this.setup();
-        const statements = sql.split("-- hack: split on this");
+        const statements = sql.split('-- hack: split on this');
         while (statements.length > 1) {
             await this.runDuckDBQuery(statements[0]);
             statements.shift();

package/dist/duckdb_wasm_connection_browser.d.ts

@@ -1,5 +1,5 @@
-import * as duckdb from "@duckdb/duckdb-wasm";
-import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from "./duckdb_wasm_connection";
+import * as duckdb from '@duckdb/duckdb-wasm';
+import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from './duckdb_wasm_connection';
 export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
     getBundles(): duckdb.DuckDBBundles;
     createHash(sqlCommand: string): Promise<string>;

package/dist/duckdb_wasm_connection_browser.js

@@ -54,11 +54,11 @@ class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection
     }
     async createHash(sqlCommand) {
         const msgUint8 = new TextEncoder().encode(sqlCommand);
-        const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
+        const hashBuffer = await crypto.subtle.digest('SHA-256', msgUint8);
         const hashArray = Array.from(new Uint8Array(hashBuffer));
         const hashHex = hashArray
-            .map((b) => b.toString(16).padStart(2, "0"))
-            .join("");
+            .map(b => b.toString(16).padStart(2, '0'))
+            .join('');
         return hashHex;
     }
 }

package/dist/duckdb_wasm_connection_node.d.ts

@@ -1,5 +1,5 @@
-import { DuckDBBundles } from "@duckdb/duckdb-wasm";
-import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from "./duckdb_wasm_connection";
+import { DuckDBBundles } from '@duckdb/duckdb-wasm';
+import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from './duckdb_wasm_connection';
 export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
     getBundles(): DuckDBBundles;
     createHash(sqlCommand: string): Promise<string>;

package/dist/duckdb_wasm_connection_node.js

@@ -30,28 +30,28 @@ const crypto_1 = __importDefault(require("crypto"));
 const duckdb_wasm_connection_1 = require("./duckdb_wasm_connection");
 class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection {
     getBundles() {
-        const resolvePath = require.resolve("@duckdb/duckdb-wasm");
+        const resolvePath = require.resolve('@duckdb/duckdb-wasm');
         if (!resolvePath) {
-            throw new Error("Unable to resolve @duckdb/duckdb-wasm path");
+            throw new Error('Unable to resolve @duckdb/duckdb-wasm path');
         }
         const distMatch = resolvePath.match(/^.*\/dist\//);
         if (!distMatch) {
-            throw new Error("Unable to resolve @duckdb/duckdb-wasm dist path");
+            throw new Error('Unable to resolve @duckdb/duckdb-wasm dist path');
         }
         const dist = distMatch[0];
         return {
-            "mvp": {
-                "mainModule": `${dist}/duckdb-mvp.wasm`,
-                "mainWorker": `${dist}/duckdb-node-mvp.worker.cjs`
+            mvp: {
+                mainModule: `${dist}/duckdb-mvp.wasm`,
+                mainWorker: `${dist}/duckdb-node-mvp.worker.cjs`,
+            },
+            eh: {
+                mainModule: `${dist}/duckdb-eh.wasm`,
+                mainWorker: `${dist}/duckdb-node-eh.worker.cjs`,
             },
-            "eh": {
-                "mainModule": `${dist}/duckdb-eh.wasm`,
-                "mainWorker": `${dist}/duckdb-node-eh.worker.cjs`
-            }
         };
     }
     async createHash(sqlCommand) {
-        return crypto_1.default.createHash("md5").update(sqlCommand).digest("hex");
+        return crypto_1.default.createHash('md5').update(sqlCommand).digest('hex');
     }
 }
 exports.DuckDBWASMConnection = DuckDBWASMConnection;

package/dist/index.d.ts

@@ -1 +1 @@
-export { DuckDBConnection } from "./duckdb_connection";
+export { DuckDBConnection } from './duckdb_connection';

package/package.json

@@ -1,9 +1,12 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.24-dev230224214106",
+  "version": "0.0.24-dev230301165401",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
+  "engines": {
+    "node": ">=11"
+  },
   "exports": {
     ".": "./dist/index.js",
     "./package.json": "./package.json",
@@ -38,7 +41,8 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.20.0",
-    "@malloydata/malloy": "^0.0.24-dev230224214106",
+    "@malloydata/malloy": "^0.0.24-dev230301165401",
+    "apache-arrow": "^9.0.0",
     "duckdb": "0.6.1",
     "web-worker": "^1.2.0"
   }