@malloydata/db-duckdb 0.0.22-dev230207183024 → 0.0.22-dev230208210335

package/dist/duckdb_wasm_connection.d.ts

@@ -1,6 +1,7 @@
 import * as duckdb from "@duckdb/duckdb-wasm";
-import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
+import { QueryDataRow, RunSQLOptions, StructDef } from "@malloydata/malloy";
 import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
+declare type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;
     private databasePath;
@@ -10,6 +11,8 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     protected _database: duckdb.AsyncDuckDB | null;
     protected isSetup: boolean;
     private worker;
+    private remoteFileCallbacks;
+    private remoteFileStatus;
     constructor(name: string, databasePath?: string | null, workingDirectory?: string, queryOptions?: QueryOptionsReader);
     private init;
     abstract getBundles(): duckdb.DuckDBBundles;
@@ -21,6 +24,12 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
         totalRows: number;
     }>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
+    fetchSchemaForTables(tables: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
     close(): Promise<void>;
+    registerRemoteTableCallback(callback: RemoteFileCallback): void;
     registerRemoteTable(tableName: string, url: string): Promise<void>;
 }
+export {};

package/dist/duckdb_wasm_connection.js

@@ -51,6 +51,7 @@ exports.DuckDBWASMConnection = void 0;
  */
 const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const web_worker_1 = __importDefault(require("web-worker"));
+const malloy_1 = require("@malloydata/malloy");
 const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 /**
@@ -126,6 +127,8 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
         this._database = null;
         this.isSetup = false;
         this.worker = null;
+        this.remoteFileCallbacks = [];
+        this.remoteFileStatus = {};
         this.connecting = this.init();
     }
     async init() {
@@ -195,6 +198,28 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             }
         }
     }
+    async fetchSchemaForTables(tables) {
+        var _a;
+        await this.setup();
+        for (const tableUri of tables) {
+            const { tablePath } = (0, malloy_1.parseTableURI)(tableUri);
+            if (tablePath.match(/^https?:\/\//)) {
+                continue;
+            }
+            if (this.remoteFileStatus[tablePath]) {
+                continue;
+            }
+            for (const callback of this.remoteFileCallbacks) {
+                this.remoteFileStatus[tablePath] = true;
+                const data = await callback(tablePath);
+                if (data) {
+                    await ((_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileBuffer(tablePath, data));
+                    break;
+                }
+            }
+        }
+        return super.fetchSchemaForTables(tables);
+    }
     async close() {
         if (this._connection) {
             await this._connection.close();
@@ -209,6 +234,9 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             this.worker = null;
         }
     }
+    registerRemoteTableCallback(callback) {
+        this.remoteFileCallbacks.push(callback);
+    }
     async registerRemoteTable(tableName, url) {
         var _a;
         (_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileURL(tableName, url, duckdb.DuckDBDataProtocol.HTTP, true);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.22-dev230207183024",
+  "version": "0.0.22-dev230208210335",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -38,7 +38,7 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.20.0",
-    "@malloydata/malloy": "^0.0.22-dev230207183024",
+    "@malloydata/malloy": "^0.0.22-dev230208210335",
     "duckdb": "0.6.1",
     "web-worker": "^1.2.0"
   }