@malloydata/db-duckdb 0.0.41-dev230615144150 → 0.0.41-dev230615181401

package/dist/duckdb_wasm_connection.d.ts

@@ -5,11 +5,11 @@ declare type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | un
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;
     private databasePath;
-    private workingDirectory;
+    protected workingDirectory: string;
     connecting: Promise<void>;
     protected _connection: duckdb.AsyncDuckDBConnection | null;
     protected _database: duckdb.AsyncDuckDB | null;
-    protected isSetup: boolean;
+    protected isSetup: Promise<void> | undefined;
     private worker;
     private remoteFileCallbacks;
     private remoteFileStatus;
@@ -18,6 +18,7 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     abstract getBundles(): duckdb.DuckDBBundles;
     get connection(): duckdb.AsyncDuckDBConnection | null;
     get database(): duckdb.AsyncDuckDB | null;
+    loadExtension(ext: string): Promise<void>;
     protected setup(): Promise<void>;
     protected runDuckDBQuery(sql: string): Promise<{
         rows: QueryDataRow[];

package/dist/duckdb_wasm_connection.js

@@ -54,7 +54,8 @@ const web_worker_1 = __importDefault(require("web-worker"));
 const malloy_1 = require("@malloydata/malloy");
 const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
-const TABLE_MATCH = /FROM\s*'(.*)'/gi;
+const TABLE_MATCH = /FROM\s*('([^']*)'|"([^"]*)")/gi;
+const TABLE_FUNCTION_MATCH = /FROM\s+[a-z0-9_]+\(('([^']*)'|"([^"]*)")/gi;
 /**
  * Arrow's toJSON() doesn't really do what I'd expect, since
  * it still includes Arrow objects like DecimalBigNums and Vectors,
@@ -94,7 +95,9 @@ const unwrapArrow = (value) => {
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
             const result = {};
             for (const key in obj) {
-                result[key] = unwrapArrow(obj[key]);
+                if (Object.prototype.hasOwnProperty.call(obj, key)) {
+                    result[key] = unwrapArrow(obj[key]);
+                }
             }
             return result;
         }
@@ -126,7 +129,6 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
         this.workingDirectory = workingDirectory;
         this._connection = null;
         this._database = null;
-        this.isSetup = false;
         this.worker = null;
         this.remoteFileCallbacks = [];
         this.remoteFileStatus = {};
@@ -164,8 +166,31 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     get database() {
         return this._database;
     }
+    async loadExtension(ext) {
+        try {
+            await this.runDuckDBQuery(`INSTALL '${ext}'`);
+            await this.runDuckDBQuery(`LOAD '${ext}'`);
+        }
+        catch (error) {
+            // eslint-disable-next-line no-console
+            console.error(`Unable to load ${ext} extension`, error);
+        }
+    }
     async setup() {
+        const doSetup = async () => {
+            if (this.workingDirectory) {
+                await this.runDuckDBQuery(`SET FILE_SEARCH_PATH='${this.workingDirectory}'`);
+            }
+            // Not quite ready for prime time
+            // for (const ext of ['json', 'httpfs', 'icu']) {
+            //   await this.loadExtension(ext);
+            // }
+        };
         await this.connecting;
+        if (!this.isSetup) {
+            this.isSetup = doSetup();
+        }
+        await this.isSetup;
     }
     async runDuckDBQuery(sql) {
         var _a;
@@ -234,7 +259,10 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     async fetchSchemaForSQLBlock(sqlRef) {
         const tables = [];
         for (const match of sqlRef.selectStr.matchAll(TABLE_MATCH)) {
-            tables.push(match[1]);
+            tables.push(match[2] || match[3]);
+        }
+        for (const match of sqlRef.selectStr.matchAll(TABLE_FUNCTION_MATCH)) {
+            tables.push(match[2] || match[3]);
         }
         await this.findTables(tables);
         return super.fetchSchemaForSQLBlock(sqlRef);

package/dist/duckdb_wasm_connection.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/duckdb_wasm_connection.spec.js

@@ -0,0 +1,92 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const duckdb_common_1 = require("./duckdb_common");
+const duckdb_wasm_connection_node_1 = require("./duckdb_wasm_connection_node");
+describe('DuckDBWasmConnection', () => {
+    let connection;
+    let findTables;
+    beforeAll(async () => {
+        connection = new duckdb_wasm_connection_node_1.DuckDBWASMConnection('duckdb');
+        await connection.runSQL('SELECT 1');
+    });
+    afterAll(async () => {
+        await connection.close();
+        await new Promise(resolve => setTimeout(resolve, 10000));
+    });
+    beforeEach(() => {
+        jest
+            .spyOn(duckdb_common_1.DuckDBCommon.prototype, 'fetchSchemaForSQLBlock')
+            .mockResolvedValue({
+            error: 'mocked',
+        });
+        findTables = jest.spyOn(connection, 'findTables');
+    });
+    afterEach(() => {
+        jest.resetAllMocks();
+    });
+    it('finds simple tables in SQL', async () => {
+        await connection.fetchSchemaForSQLBlock({
+            selectStr: `
+SELECT
+  created_at,
+  sale_price,
+  inventory_item_id
+FROM 'order_items.parquet'
+SELECT
+id,
+  product_department,
+  product_category,
+  created_at AS inventory_items_created_at
+FROM "inventory_items.parquet"
+`,
+        });
+        expect(findTables).toHaveBeenCalledWith([
+            'order_items.parquet',
+            'inventory_items.parquet',
+        ]);
+    });
+    it('finds table functions in SQL', async () => {
+        await connection.fetchSchemaForSQLBlock({
+            selectStr: `
+SELECT
+  created_at,
+  sale_price,
+  inventory_item_id
+FROM read_parquet('order_items2.parquet', arg='value')
+SELECT
+id,
+  product_department,
+  product_category,
+  created_at AS inventory_items_created_at
+FROM read_parquet("inventory_items2.parquet")
+`,
+        });
+        expect(findTables).toHaveBeenCalledWith([
+            'order_items2.parquet',
+            'inventory_items2.parquet',
+        ]);
+    });
+});
+//# sourceMappingURL=duckdb_wasm_connection.spec.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.41-dev230615144150",
+  "version": "0.0.41-dev230615181401",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -40,10 +40,10 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@malloydata/duckdb-wasm": "0.0.1",
-    "@malloydata/malloy": "^0.0.41-dev230615144150",
+    "@malloydata/duckdb-wasm": "0.0.2",
+    "@malloydata/malloy": "^0.0.40",
     "apache-arrow": "^11.0.0",
-    "duckdb": "0.7.1",
+    "duckdb": "0.8.1",
     "web-worker": "^1.2.0"
   }
 }