@malloydata/db-duckdb 0.0.19 → 0.0.20-dev230110224239

package/dist/duckdb_common.d.ts

@@ -54,6 +54,7 @@ export declare abstract class DuckDBCommon implements Connection, PersistSQLResu
     private getTableSchema;
     canStream(): this is StreamingConnection;
     test(): Promise<void>;
-    protected abstract createHash(sqlCommand: string): Promise<string>;
+    abstract createHash(sqlCommand: string): Promise<string>;
     manifestTemporaryTable(sqlCommand: string): Promise<string>;
+    abstract close(): Promise<void>;
 }

package/dist/duckdb_connection.d.ts

@@ -1,13 +1,16 @@
 import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
-import { Row } from "duckdb";
+import { Connection, Database, Row } from "duckdb";
 import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
 export declare class DuckDBConnection extends DuckDBCommon {
     readonly name: string;
+    private databasePath;
     private workingDirectory;
-    protected connection: any;
-    protected database: any;
+    connecting: Promise<void>;
+    protected connection: Connection | null;
+    protected database: Database | null;
     protected isSetup: Promise<void> | undefined;
     constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
+    private init;
     protected setup(): Promise<void>;
     protected runDuckDBQuery(sql: string): Promise<{
         rows: Row[];
@@ -15,4 +18,5 @@ export declare class DuckDBConnection extends DuckDBCommon {
     }>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
     createHash(sqlCommand: string): Promise<string>;
+    close(): Promise<void>;
 }

package/dist/duckdb_connection.js

@@ -11,46 +11,35 @@
  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  * GNU General Public License for more details.
  */
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBConnection = void 0;
-const crypto = __importStar(require("crypto"));
+const crypto_1 = __importDefault(require("crypto"));
 const duckdb_common_1 = require("./duckdb_common");
 const duckdb_1 = require("duckdb");
 class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     constructor(name, databasePath = ":memory:", workingDirectory = ".", queryOptions) {
         super(queryOptions);
         this.name = name;
+        this.databasePath = databasePath;
         this.workingDirectory = workingDirectory;
-        this.database = new duckdb_1.Database(databasePath, duckdb_1.OPEN_READWRITE, // databasePath === ":memory:" ? OPEN_READWRITE : OPEN_READONLY,
-        (err) => {
-            if (err) {
-                return console.error(err);
-            }
+        this.connection = null;
+        this.database = null;
+        this.connecting = this.init();
+    }
+    async init() {
+        return new Promise((resolve, reject) => {
+            this.database = new duckdb_1.Database(this.databasePath, duckdb_1.OPEN_READWRITE, // databasePath === ":memory:" ? OPEN_READWRITE : OPEN_READONLY,
+            (err) => {
+                if (err) {
+                    reject(err);
+                }
+            });
+            this.connection = this.database.connect();
+            resolve();
         });
-        this.connection = this.database.connect();
     }
     async setup() {
         const doSetup = async () => {
@@ -74,6 +63,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
                 console.error("Unable to load httpfs extension", error);
             }
         };
+        await this.connecting;
         if (!this.isSetup) {
             this.isSetup = doSetup();
         }
@@ -81,21 +71,29 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
     }
     async runDuckDBQuery(sql) {
         return new Promise((resolve, reject) => {
-            this.connection.all(sql, (err, rows) => {
-                if (err) {
-                    reject(err);
-                }
-                else {
-                    resolve({
-                        rows,
-                        totalRows: rows.length,
-                    });
-                }
-            });
+            if (this.connection) {
+                this.connection.all(sql, (err, rows) => {
+                    if (err) {
+                        reject(err);
+                    }
+                    else {
+                        resolve({
+                            rows,
+                            totalRows: rows.length,
+                        });
+                    }
+                });
+            }
+            else {
+                reject(new Error("Connection not open"));
+            }
         });
     }
     async *runSQLStream(sql, _options = {}) {
         await this.setup();
+        if (!this.connection) {
+            throw new Error("Connection not open");
+        }
         const statements = sql.split("-- hack: split on this");
         while (statements.length > 1) {
             await this.runDuckDBQuery(statements[0]);
@@ -105,8 +103,17 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
             yield row;
         }
     }
-    createHash(sqlCommand) {
-        return Promise.resolve(crypto.createHash("md5").update(sqlCommand).digest("hex"));
+    async createHash(sqlCommand) {
+        return crypto_1.default.createHash("md5").update(sqlCommand).digest("hex");
+    }
+    async close() {
+        if (this.connection) {
+            this.connection = null;
+        }
+        if (this.database) {
+            this.database.close();
+            this.database = null;
+        }
     }
 }
 exports.DuckDBConnection = DuckDBConnection;

package/dist/duckdb_wasm_connection.d.ts

@@ -1,15 +1,18 @@
-import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
 import * as duckdb from "@duckdb/duckdb-wasm";
+import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
 import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
-export declare class DuckDBWASMConnection extends DuckDBCommon {
+export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;
+    private databasePath;
     private workingDirectory;
     connecting: Promise<void>;
     protected _connection: duckdb.AsyncDuckDBConnection | null;
     protected _database: duckdb.AsyncDuckDB | null;
     protected isSetup: boolean;
-    constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
+    private worker;
+    constructor(name: string, databasePath?: string | null, workingDirectory?: string, queryOptions?: QueryOptionsReader);
     private init;
+    abstract getBundles(): duckdb.DuckDBBundles;
     get connection(): duckdb.AsyncDuckDBConnection | null;
     get database(): duckdb.AsyncDuckDB | null;
     protected setup(): Promise<void>;
@@ -18,5 +21,6 @@ export declare class DuckDBWASMConnection extends DuckDBCommon {
         totalRows: number;
     }>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
-    protected createHash(sqlCommand: string): Promise<string>;
+    close(): Promise<void>;
+    registerRemoteTable(tableName: string, url: string): Promise<void>;
 }

package/dist/duckdb_wasm_connection.js

@@ -22,9 +22,25 @@ var __importStar = (this && this.__importStar) || function (mod) {
     __setModuleDefault(result, mod);
     return result;
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBWASMConnection = void 0;
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
 const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
+const web_worker_1 = __importDefault(require("web-worker"));
 const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 /**
@@ -89,29 +105,38 @@ const unwrapRow = (row) => {
 const unwrapTable = (table) => {
     return table.toArray().map(unwrapRow);
 };
+const isNode = () => typeof navigator === "undefined";
 class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
-    constructor(name, databasePath = "test/data/duckdb/duckdb_test.db", workingDirectory = "/", queryOptions) {
+    constructor(name, databasePath = null, workingDirectory = "/", queryOptions) {
         super(queryOptions);
         this.name = name;
+        this.databasePath = databasePath;
         this.workingDirectory = workingDirectory;
         this._connection = null;
         this._database = null;
         this.isSetup = false;
+        this.worker = null;
         this.connecting = this.init();
     }
     async init() {
-        const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
         // Select a bundle based on browser checks
-        const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+        const bundle = await duckdb.selectBundle(this.getBundles());
         if (bundle.mainWorker) {
-            const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], {
-                type: "text/javascript",
-            }));
+            const workerUrl = isNode()
+                ? bundle.mainWorker
+                : URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], {
+                    type: "text/javascript",
+                }));
             // Instantiate the asynchronous version of DuckDB-wasm
-            const worker = new Worker(workerUrl);
-            const logger = new duckdb.ConsoleLogger();
-            this._database = new duckdb.AsyncDuckDB(logger, worker);
+            this.worker = new web_worker_1.default(workerUrl);
+            const logger = new duckdb.VoidLogger();
+            this._database = new duckdb.AsyncDuckDB(logger, this.worker);
             await this._database.instantiate(bundle.mainModule, bundle.pthreadWorker);
+            if (this.databasePath) {
+                await this._database.open({
+                    path: this.databasePath,
+                });
+            }
             URL.revokeObjectURL(workerUrl);
             this._connection = await this._database.connect();
         }
@@ -160,14 +185,23 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             }
         }
     }
-    async createHash(sqlCommand) {
-        const msgUint8 = new TextEncoder().encode(sqlCommand);
-        const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
-        const hashArray = Array.from(new Uint8Array(hashBuffer));
-        const hashHex = hashArray
-            .map((b) => b.toString(16).padStart(2, "0"))
-            .join("");
-        return hashHex;
+    async close() {
+        if (this._connection) {
+            await this._connection.close();
+            this._connection = null;
+        }
+        if (this._database) {
+            await this._database.terminate();
+            this._database = null;
+        }
+        if (this.worker) {
+            this.worker.terminate();
+            this.worker = null;
+        }
+    }
+    async registerRemoteTable(tableName, url) {
+        var _a;
+        (_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileURL(tableName, url, duckdb.DuckDBDataProtocol.HTTP, true);
     }
 }
 exports.DuckDBWASMConnection = DuckDBWASMConnection;

package/dist/duckdb_wasm_connection_browser.d.ts

@@ -0,0 +1,6 @@
+import * as duckdb from "@duckdb/duckdb-wasm";
+import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from "./duckdb_wasm_connection";
+export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
+    getBundles(): duckdb.DuckDBBundles;
+    createHash(sqlCommand: string): Promise<string>;
+}

package/dist/duckdb_wasm_connection_browser.js

@@ -0,0 +1,56 @@
+"use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBWASMConnection = void 0;
+const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
+const duckdb_wasm_connection_1 = require("./duckdb_wasm_connection");
+class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection {
+    getBundles() {
+        return duckdb.getJsDelivrBundles();
+    }
+    async createHash(sqlCommand) {
+        const msgUint8 = new TextEncoder().encode(sqlCommand);
+        const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
+        const hashArray = Array.from(new Uint8Array(hashBuffer));
+        const hashHex = hashArray
+            .map((b) => b.toString(16).padStart(2, "0"))
+            .join("");
+        return hashHex;
+    }
+}
+exports.DuckDBWASMConnection = DuckDBWASMConnection;
+//# sourceMappingURL=duckdb_wasm_connection_browser.js.map

package/dist/duckdb_wasm_connection_node.d.ts

@@ -0,0 +1,6 @@
+import { DuckDBBundles } from "@duckdb/duckdb-wasm";
+import { DuckDBWASMConnection as DuckDBWASMConnectionBase } from "./duckdb_wasm_connection";
+export declare class DuckDBWASMConnection extends DuckDBWASMConnectionBase {
+    getBundles(): DuckDBBundles;
+    createHash(sqlCommand: string): Promise<string>;
+}

package/dist/duckdb_wasm_connection_node.js

@@ -0,0 +1,48 @@
+"use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBWASMConnection = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const duckdb_wasm_connection_1 = require("./duckdb_wasm_connection");
+class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection {
+    getBundles() {
+        const resolvePath = require.resolve("@duckdb/duckdb-wasm");
+        if (!resolvePath) {
+            throw new Error("Unable to resolve @duckdb/duckdb-wasm path");
+        }
+        const distMatch = resolvePath.match(/^.*\/dist\//);
+        if (!distMatch) {
+            throw new Error("Unable to resolve @duckdb/duckdb-wasm dist path");
+        }
+        const dist = distMatch[0];
+        return {
+            mvp: {
+                mainModule: `${dist}/duckdb-mvp.wasm`,
+                mainWorker: `${dist}/duckdb-node-mvp.worker.cjs`,
+            },
+            eh: {
+                mainModule: `${dist}/duckdb-eh.wasm`,
+                mainWorker: `${dist}/duckdb-node-eh.worker.cjs`,
+            },
+        };
+    }
+    async createHash(sqlCommand) {
+        return crypto_1.default.createHash("md5").update(sqlCommand).digest("hex");
+    }
+}
+exports.DuckDBWASMConnection = DuckDBWASMConnection;
+//# sourceMappingURL=duckdb_wasm_connection_node.js.map

package/package.json

@@ -1,9 +1,27 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.19",
+  "version": "0.0.20-dev230110224239",
   "license": "GPL-2.0",
-  "main": "dist/index.js",
-  "types": "dist/index.d.ts",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": "./dist/index.js",
+    "./package.json": "./package.json",
+    "./wasm": {
+      "browser": "./dist/duckdb_wasm_connection_browser.js",
+      "node": "./dist/duckdb_wasm_connection_node.js"
+    }
+  },
+  "typesVersions": {
+    "*": {
+      "index": [
+        "./dist/index.d.ts"
+      ],
+      "wasm": [
+        "./dist/duckdb_wasm_connection_browser.d.ts"
+      ]
+    }
+  },
   "homepage": "https://github.com/malloydata/malloy#readme",
   "repository": {
     "type": "git",
@@ -18,8 +36,9 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@duckdb/duckdb-wasm": "^1.17.0",
-    "@malloydata/malloy": "^0.0.19",
-    "duckdb": "0.6.1"
+    "@duckdb/duckdb-wasm": "^1.20.0",
+    "@malloydata/malloy": "^0.0.20-dev230110224239",
+    "duckdb": "0.6.1",
+    "web-worker": "^1.2.0"
   }
 }