@malloydata/db-duckdb 0.0.22-dev230207182613 → 0.0.22-dev230208192220

package/dist/duckdb_common.d.ts

@@ -1,4 +1,4 @@
-import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, RunSQLOptions, SQLBlock, StreamingConnection, StructDef, QueryDataRow } from "@malloydata/malloy";
+import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from "@malloydata/malloy";
 export interface DuckDBQueryOptions {
     rowLimit: number;
 }

package/dist/duckdb_common.js

@@ -25,15 +25,15 @@ exports.DuckDBCommon = void 0;
  */
 const malloy_1 = require("@malloydata/malloy");
 const duckDBToMalloyTypes = {
-    BIGINT: "number",
-    DOUBLE: "number",
-    VARCHAR: "string",
-    DATE: "date",
-    TIMESTAMP: "timestamp",
-    TIME: "string",
-    DECIMAL: "number",
-    BOOLEAN: "boolean",
-    INTEGER: "number",
+    "BIGINT": "number",
+    "DOUBLE": "number",
+    "VARCHAR": "string",
+    "DATE": "date",
+    "TIMESTAMP": "timestamp",
+    "TIME": "string",
+    "DECIMAL": "number",
+    "BOOLEAN": "boolean",
+    "INTEGER": "number"
 };
 class DuckDBCommon {
     constructor(queryOptions) {
@@ -83,23 +83,23 @@ class DuckDBCommon {
         if (result.length > rowLimit) {
             result = result.slice(0, rowLimit);
         }
-        return { rows: result, totalRows: result.length };
+        return { "rows": result, "totalRows": result.length };
     }
     async getSQLBlockSchema(sqlRef) {
         const structDef = {
-            type: "struct",
-            dialect: "duckdb",
-            name: sqlRef.name,
-            structSource: {
-                type: "sql",
-                method: "subquery",
-                sqlBlock: sqlRef,
+            "type": "struct",
+            "dialect": "duckdb",
+            "name": sqlRef.name,
+            "structSource": {
+                "type": "sql",
+                "method": "subquery",
+                "sqlBlock": sqlRef
             },
-            structRelationship: {
-                type: "basetable",
-                connectionName: this.name,
+            "structRelationship": {
+                "type": "basetable",
+                "connectionName": this.name
             },
-            fields: [],
+            "fields": []
         };
         await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.selectStr})`, structDef);
         return structDef;
@@ -174,16 +174,16 @@ class DuckDBCommon {
             if (structMatch && structMatch.groups) {
                 const newTypeMap = this.stringToTypeMap(structMatch.groups["fields"]);
                 const innerStructDef = {
-                    type: "struct",
+                    "type": "struct",
                     name,
-                    dialect: this.dialectName,
-                    structSource: { type: arrayMatch ? "nested" : "inline" },
-                    structRelationship: {
-                        type: arrayMatch ? "nested" : "inline",
-                        field: name,
-                        isArray: false,
+                    "dialect": this.dialectName,
+                    "structSource": { "type": arrayMatch ? "nested" : "inline" },
+                    "structRelationship": {
+                        "type": arrayMatch ? "nested" : "inline",
+                        "field": name,
+                        "isArray": false
                     },
-                    fields: [],
+                    "fields": []
                 };
                 this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
                 structDef.fields.push(innerStructDef);
@@ -192,27 +192,31 @@ class DuckDBCommon {
                 if (arrayMatch) {
                     malloyType = duckDBToMalloyTypes[duckDBType];
                     const innerStructDef = {
-                        type: "struct",
+                        "type": "struct",
                         name,
-                        dialect: this.dialectName,
-                        structSource: { type: "nested" },
-                        structRelationship: { type: "nested", field: name, isArray: true },
-                        fields: [{ type: malloyType, name: "value" }],
+                        "dialect": this.dialectName,
+                        "structSource": { "type": "nested" },
+                        "structRelationship": {
+                            "type": "nested",
+                            "field": name,
+                            "isArray": true
+                        },
+                        "fields": [{ "type": malloyType, "name": "value" }]
                     };
                     structDef.fields.push(innerStructDef);
                 }
                 else {
                     if (malloyType !== undefined) {
                         structDef.fields.push({
-                            type: malloyType,
-                            name,
+                            "type": malloyType,
+                            name
                         });
                     }
                     else {
                         structDef.fields.push({
                             name,
-                            type: "string",
-                            e: [`'DuckDB type "${duckDBType}" not supported by Malloy'`],
+                            "type": "string",
+                            "e": [`'DuckDB type "${duckDBType}" not supported by Malloy'`]
                         });
                     }
                 }
@@ -233,11 +237,11 @@ class DuckDBCommon {
         if (!inCache) {
             try {
                 inCache = {
-                    structDef: await this.getSQLBlockSchema(sqlRef),
+                    "structDef": await this.getSQLBlockSchema(sqlRef)
                 };
             }
             catch (error) {
-                inCache = { error: error.message };
+                inCache = { "error": error.message };
             }
             this.sqlSchemaCache.set(key, inCache);
         }
@@ -251,12 +255,12 @@ class DuckDBCommon {
             if (!inCache) {
                 try {
                     inCache = {
-                        schema: await this.getTableSchema(tableURL),
+                        "schema": await this.getTableSchema(tableURL)
                     };
                     this.schemaCache.set(tableURL, inCache);
                 }
                 catch (error) {
-                    inCache = { error: error.message };
+                    inCache = { "error": error.message };
                 }
             }
             if (inCache.schema !== undefined) {
@@ -271,15 +275,15 @@ class DuckDBCommon {
     async getTableSchema(tableURL) {
         const { tablePath } = (0, malloy_1.parseTableURI)(tableURL);
         const structDef = {
-            type: "struct",
-            name: tablePath,
-            dialect: "duckdb",
-            structSource: { type: "table", tablePath },
-            structRelationship: {
-                type: "basetable",
-                connectionName: this.name,
+            "type": "struct",
+            "name": tablePath,
+            "dialect": "duckdb",
+            "structSource": { "type": "table", tablePath },
+            "structRelationship": {
+                "type": "basetable",
+                "connectionName": this.name
             },
-            fields: [],
+            "fields": []
         };
         const quotedTablePath = tablePath.match(/[:*/]/)
             ? `'${tablePath}'`
@@ -305,6 +309,6 @@ class DuckDBCommon {
 }
 exports.DuckDBCommon = DuckDBCommon;
 DuckDBCommon.DEFAULT_QUERY_OPTIONS = {
-    rowLimit: 10,
+    "rowLimit": 10
 };
 //# sourceMappingURL=duckdb_common.js.map

package/dist/duckdb_connection.js

@@ -89,7 +89,7 @@ class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
                     else {
                         resolve({
                             rows,
-                            totalRows: rows.length,
+                            "totalRows": rows.length
                         });
                     }
                 });

package/dist/duckdb_wasm_connection.d.ts

@@ -1,6 +1,7 @@
 import * as duckdb from "@duckdb/duckdb-wasm";
-import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
+import { QueryDataRow, RunSQLOptions, StructDef } from "@malloydata/malloy";
 import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
+declare type RemoteFileCallback = (tableName: string) => Promise<Uint8Array | undefined>;
 export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     readonly name: string;
     private databasePath;
@@ -10,6 +11,8 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
     protected _database: duckdb.AsyncDuckDB | null;
     protected isSetup: boolean;
     private worker;
+    private remoteFileCallbacks;
+    private remoteFileStatus;
     constructor(name: string, databasePath?: string | null, workingDirectory?: string, queryOptions?: QueryOptionsReader);
     private init;
     abstract getBundles(): duckdb.DuckDBBundles;
@@ -21,6 +24,12 @@ export declare abstract class DuckDBWASMConnection extends DuckDBCommon {
         totalRows: number;
     }>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
+    fetchSchemaForTables(tables: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
     close(): Promise<void>;
+    registerRemoteTableCallback(callback: RemoteFileCallback): void;
     registerRemoteTable(tableName: string, url: string): Promise<void>;
 }
+export {};

package/dist/duckdb_wasm_connection.js

@@ -51,6 +51,7 @@ exports.DuckDBWASMConnection = void 0;
  */
 const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
 const web_worker_1 = __importDefault(require("web-worker"));
+const malloy_1 = require("@malloydata/malloy");
 const apache_arrow_1 = require("apache-arrow");
 const duckdb_common_1 = require("./duckdb_common");
 /**
@@ -126,6 +127,8 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
         this._database = null;
         this.isSetup = false;
         this.worker = null;
+        this.remoteFileCallbacks = [];
+        this.remoteFileStatus = {};
         this.connecting = this.init();
     }
     async init() {
@@ -135,7 +138,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             const workerUrl = isNode()
                 ? bundle.mainWorker
                 : URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], {
-                    type: "text/javascript",
+                    "type": "text/javascript"
                 }));
             // Instantiate the asynchronous version of DuckDB-wasm
             this.worker = new web_worker_1.default(workerUrl);
@@ -144,7 +147,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             await this._database.instantiate(bundle.mainModule, bundle.pthreadWorker);
             if (this.databasePath) {
                 await this._database.open({
-                    path: this.databasePath,
+                    "path": this.databasePath
                 });
             }
             URL.revokeObjectURL(workerUrl);
@@ -172,7 +175,7 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             return {
                 // Normalize the data from its default proxied form
                 rows,
-                totalRows: table.numRows,
+                "totalRows": table.numRows
             };
         }
         else {
@@ -195,6 +198,28 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             }
         }
     }
+    async fetchSchemaForTables(tables) {
+        var _a;
+        await this.setup();
+        for (const tableUri of tables) {
+            const { tablePath } = (0, malloy_1.parseTableURI)(tableUri);
+            if (tablePath.match(/^https?:\/\//)) {
+                continue;
+            }
+            if (this.remoteFileStatus[tablePath]) {
+                continue;
+            }
+            for (const callback of this.remoteFileCallbacks) {
+                this.remoteFileStatus[tablePath] = true;
+                const data = await callback(tablePath);
+                if (data) {
+                    await ((_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileBuffer(tablePath, data));
+                    break;
+                }
+            }
+        }
+        return super.fetchSchemaForTables(tables);
+    }
     async close() {
         if (this._connection) {
             await this._connection.close();
@@ -209,6 +234,9 @@ class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
             this.worker = null;
         }
     }
+    registerRemoteTableCallback(callback) {
+        this.remoteFileCallbacks.push(callback);
+    }
     async registerRemoteTable(tableName, url) {
         var _a;
         (_a = this.database) === null || _a === void 0 ? void 0 : _a.registerFileURL(tableName, url, duckdb.DuckDBDataProtocol.HTTP, true);

package/dist/duckdb_wasm_connection_node.js

@@ -40,14 +40,14 @@ class DuckDBWASMConnection extends duckdb_wasm_connection_1.DuckDBWASMConnection
         }
         const dist = distMatch[0];
         return {
-            mvp: {
-                mainModule: `${dist}/duckdb-mvp.wasm`,
-                mainWorker: `${dist}/duckdb-node-mvp.worker.cjs`,
-            },
-            eh: {
-                mainModule: `${dist}/duckdb-eh.wasm`,
-                mainWorker: `${dist}/duckdb-node-eh.worker.cjs`,
+            "mvp": {
+                "mainModule": `${dist}/duckdb-mvp.wasm`,
+                "mainWorker": `${dist}/duckdb-node-mvp.worker.cjs`
             },
+            "eh": {
+                "mainModule": `${dist}/duckdb-eh.wasm`,
+                "mainWorker": `${dist}/duckdb-node-eh.worker.cjs`
+            }
         };
     }
     async createHash(sqlCommand) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.22-dev230207182613",
+  "version": "0.0.22-dev230208192220",
   "license": "MIT",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
@@ -38,7 +38,7 @@
   },
   "dependencies": {
     "@duckdb/duckdb-wasm": "^1.20.0",
-    "@malloydata/malloy": "^0.0.22-dev230207182613",
+    "@malloydata/malloy": "^0.0.22-dev230208192220",
     "duckdb": "0.6.1",
     "web-worker": "^1.2.0"
   }