@malloydata/db-duckdb 0.0.96-dev231025212541 → 0.0.96-dev231025215824

package/dist/duckdb.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/duckdb.spec.js

@@ -0,0 +1,116 @@
+"use strict";
+/*
+ * Copyright 2023 Google LLC
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files
+ * (the "Software"), to deal in the Software without restriction,
+ * including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software,
+ * and to permit persons to whom the Software is furnished to do so,
+ * subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+const duckdb_common_1 = require("./duckdb_common");
+const duckdb_connection_1 = require("./duckdb_connection");
+/*
+ * !IMPORTANT
+ *
+ * The connection is reused for each test, so if you do not name your tables
+ * and keys uniquely for each test you will see cross test interactions.
+ */
+describe('DuckDBConnection', () => {
+    let connection;
+    let runRawSQL;
+    beforeAll(async () => {
+        connection = new duckdb_connection_1.DuckDBConnection('duckdb');
+        await connection.runSQL('SELECT 1');
+    });
+    afterAll(async () => {
+        await connection.close();
+    });
+    beforeEach(async () => {
+        runRawSQL = jest
+            .spyOn(duckdb_common_1.DuckDBCommon.prototype, 'runRawSQL')
+            .mockResolvedValue({ rows: [], totalRows: 0 });
+    });
+    afterEach(() => {
+        jest.resetAllMocks();
+    });
+    it('caches table schema', async () => {
+        await connection.fetchSchemaForTables({ 'test1': 'table1' }, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+        await new Promise(resolve => setTimeout(resolve));
+        await connection.fetchSchemaForTables({ 'test1': 'table1' }, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+    });
+    it('refreshes table schema', async () => {
+        await connection.fetchSchemaForTables({ 'test2': 'table2' }, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+        await new Promise(resolve => setTimeout(resolve));
+        await connection.fetchSchemaForTables({ 'test2': 'table2' }, { refreshTimestamp: Date.now() });
+        expect(runRawSQL).toBeCalledTimes(2);
+    });
+    it('caches sql schema', async () => {
+        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+        await new Promise(resolve => setTimeout(resolve));
+        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+    });
+    it('refreshes sql schema', async () => {
+        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {});
+        expect(runRawSQL).toBeCalledTimes(1);
+        await new Promise(resolve => setTimeout(resolve));
+        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {
+            refreshTimestamp: Date.now(),
+        });
+        expect(runRawSQL).toBeCalledTimes(2);
+    });
+});
+const SQL_BLOCK_1 = {
+    type: 'sqlBlock',
+    name: 'block1',
+    selectStr: `
+SELECT
+created_at,
+sale_price,
+inventory_item_id
+FROM 'order_items.parquet'
+SELECT
+id,
+product_department,
+product_category,
+created_at AS inventory_items_created_at
+FROM "inventory_items.parquet"
+`,
+};
+const SQL_BLOCK_2 = {
+    type: 'sqlBlock',
+    name: 'block2',
+    selectStr: `
+SELECT
+created_at,
+sale_price,
+inventory_item_id
+FROM read_parquet('order_items2.parquet', arg='value')
+SELECT
+id,
+product_department,
+product_category,
+created_at AS inventory_items_created_at
+FROM read_parquet("inventory_items2.parquet")
+`,
+};
+//# sourceMappingURL=duckdb.spec.js.map

package/dist/duckdb_common.d.ts

@@ -1,4 +1,5 @@
 import { MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, QueryRunStats, RunSQLOptions, SQLBlock, StreamingConnection, StructDef, TestableConnection } from '@malloydata/malloy';
+import { FetchSchemaOptions } from '@malloydata/malloy-interfaces';
 export interface DuckDBQueryOptions {
     rowLimit: number;
 }
@@ -42,14 +43,14 @@ export declare abstract class DuckDBCommon implements TestableConnection, Persis
     private stringToTypeMap;
     private fillStructDefFromTypeMap;
     private schemaFromQuery;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock): Promise<{
+    fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
         structDef: StructDef;
         error?: undefined;
     } | {
         error: string;
         structDef?: undefined;
     }>;
-    fetchSchemaForTables(tables: Record<string, string>): Promise<{
+    fetchSchemaForTables(tables: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
         schemas: Record<string, StructDef>;
         errors: Record<string, string>;
     }>;

package/dist/duckdb_common.js

@@ -230,37 +230,43 @@ class DuckDBCommon {
         }
         this.fillStructDefFromTypeMap(structDef, typeMap);
     }
-    async fetchSchemaForSQLBlock(sqlRef) {
+    async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
         const key = sqlRef.name;
         let inCache = this.sqlSchemaCache.get(key);
-        if (!inCache) {
+        if (!inCache ||
+            (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
+            const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
             try {
                 inCache = {
                     structDef: await this.getSQLBlockSchema(sqlRef),
+                    timestamp,
                 };
             }
             catch (error) {
-                inCache = { error: error.message };
+                inCache = { error: error.message, timestamp };
             }
             this.sqlSchemaCache.set(key, inCache);
         }
         return inCache;
     }
-    async fetchSchemaForTables(tables) {
+    async fetchSchemaForTables(tables, { refreshTimestamp }) {
         const schemas = {};
         const errors = {};
         for (const tableKey in tables) {
             let inCache = this.schemaCache.get(tableKey);
-            if (!inCache) {
+            if (!inCache ||
+                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
+                const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
                 const tablePath = tables[tableKey];
                 try {
                     inCache = {
                         schema: await this.getTableSchema(tableKey, tablePath),
+                        timestamp,
                     };
                     this.schemaCache.set(tableKey, inCache);
                 }
                 catch (error) {
-                    inCache = { error: error.message };
+                    inCache = { error: error.message, timestamp };
                 }
             }
             if (inCache.schema !== undefined) {

package/dist/duckdb_wasm_connection.d.ts

@@ -1,6 +1,7 @@
 import * as duckdb from '@malloydata/duckdb-wasm';
 import { QueryDataRow, RunSQLOptions, StructDef, SQLBlock } from '@malloydata/malloy';
 import { DuckDBCommon, QueryOptionsReader } from './duckdb_common';
+import { FetchSchemaOptions } from '@malloydata/malloy-interfaces';
 type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;
@@ -26,14 +27,14 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     }>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
     private findTables;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock): Promise<{
+    fetchSchemaForSQLBlock(sqlRef: SQLBlock, options: FetchSchemaOptions): Promise<{
         structDef: StructDef;
         error?: undefined;
     } | {
         error: string;
         structDef?: undefined;
     }>;
-    fetchSchemaForTables(missing: Record<string, string>): Promise<{
+    fetchSchemaForTables(missing: Record<string, string>, options: FetchSchemaOptions): Promise<{
         schemas: Record<string, StructDef>;
         errors: Record<string, string>;
     }>;

package/dist/duckdb_wasm_connection.js

@@ -226,7 +226,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             }
         }
     }
-    async findTables(tables) {
+    async findTables(tables, { refreshTimestamp }) {
         const fetchRemoteFile = async (tablePath) => {
             var _a;
             for (const callback of this.remoteFileCallbacks) {
@@ -236,7 +236,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
                     break;
                 }
             }
-            return true;
+            return refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
         };
         await this.setup();
         for (const tablePath of tables) {
@@ -248,14 +248,15 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
                 continue;
             }
             // If we're not trying to fetch start trying
-            if (!(tablePath in this.remoteFileStatus)) {
+            const mapped = this.remoteFileStatus[tablePath];
+            if (!mapped || (refreshTimestamp && refreshTimestamp > (await mapped))) {
                 this.remoteFileStatus[tablePath] = fetchRemoteFile(tablePath);
             }
             // Wait for response
             await this.remoteFileStatus[tablePath];
         }
     }
-    async fetchSchemaForSQLBlock(sqlRef) {
+    async fetchSchemaForSQLBlock(sqlRef, options) {
         const tables = [];
         for (const match of sqlRef.selectStr.matchAll(TABLE_MATCH)) {
             tables.push(match[2] || match[3]);
@@ -263,13 +264,13 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
         for (const match of sqlRef.selectStr.matchAll(TABLE_FUNCTION_MATCH)) {
             tables.push(match[2] || match[3]);
         }
-        await this.findTables(tables);
-        return super.fetchSchemaForSQLBlock(sqlRef);
+        await this.findTables(tables, options);
+        return super.fetchSchemaForSQLBlock(sqlRef, options);
     }
-    async fetchSchemaForTables(missing) {
+    async fetchSchemaForTables(missing, options) {
         const tables = Object.values(missing);
-        await this.findTables(tables);
-        return super.fetchSchemaForTables(missing);
+        await this.findTables(tables, options);
+        return super.fetchSchemaForTables(missing, options);
     }
     async close() {
         if (this._connection) {
@@ -290,7 +291,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
     }
     async registerRemoteTable(tableName, url) {
         var _a;
-        this.remoteFileStatus[tableName] = Promise.resolve(true);
+        this.remoteFileStatus[tableName] = Promise.resolve(Number.MIN_SAFE_INTEGER);
         (_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileURL(tableName, url, duckdb.DuckDBDataProtocol.HTTP, true);
     }
 }

package/dist/duckdb_wasm_connection.spec.js

@@ -62,11 +62,8 @@ id,
   created_at AS inventory_items_created_at
 FROM "inventory_items.parquet"
 `,
-        });
-        expect(findTables).toHaveBeenCalledWith([
-            'order_items.parquet',
-            'inventory_items.parquet',
-        ]);
+        }, {});
+        expect(findTables).toHaveBeenCalledWith(['order_items.parquet', 'inventory_items.parquet'], {});
     });
     it('finds table functions in SQL', async () => {
         await connection.fetchSchemaForSQLBlock({
@@ -83,11 +80,8 @@ id,
   created_at AS inventory_items_created_at
 FROM read_parquet("inventory_items2.parquet")
 `,
-        });
-        expect(findTables).toHaveBeenCalledWith([
-            'order_items2.parquet',
-            'inventory_items2.parquet',
-        ]);
+        }, {});
+        expect(findTables).toHaveBeenCalledWith(['order_items2.parquet', 'inventory_items2.parquet'], {});
     });
 });
 //# sourceMappingURL=duckdb_wasm_connection.spec.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.96-dev231025212541",
+  "version": "0.0.96-dev231025215824",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -41,7 +41,8 @@
   },
   "dependencies": {
     "@malloydata/duckdb-wasm": "0.0.2-rev3",
-    "@malloydata/malloy": "^0.0.96-dev231025212541",
+    "@malloydata/malloy": "^0.0.96-dev231025215824",
+    "@malloydata/malloy-interfaces": "^0.0.96-dev231025215824",
     "apache-arrow": "^11.0.0",
     "duckdb": "0.8.1",
     "web-worker": "^1.2.0"