@malloydata/db-duckdb 0.0.1 → 0.0.4

package/dist/duckdb_common.d.ts

@@ -0,0 +1,60 @@
+import { Connection, FetchSchemaAndRunSimultaneously, FetchSchemaAndRunStreamSimultaneously, MalloyQueryData, PersistSQLResults, PooledConnection, RunSQLOptions, SQLBlock, StreamingConnection, StructDef, QueryDataRow } from "@malloydata/malloy";
+export interface DuckDBQueryOptions {
+    rowLimit: number;
+}
+export declare type QueryOptionsReader = Partial<DuckDBQueryOptions> | (() => Partial<DuckDBQueryOptions>);
+export declare abstract class DuckDBCommon implements Connection, PersistSQLResults, StreamingConnection {
+    private queryOptions?;
+    static DEFAULT_QUERY_OPTIONS: DuckDBQueryOptions;
+    readonly name: string;
+    get dialectName(): string;
+    private readQueryOptions;
+    constructor(queryOptions?: QueryOptionsReader | undefined);
+    isPool(): this is PooledConnection;
+    canPersist(): this is PersistSQLResults;
+    protected abstract setup(): Promise<void>;
+    protected abstract runDuckDBQuery(sql: string): Promise<{
+        rows: QueryDataRow[];
+        totalRows: number;
+    }>;
+    runRawSQL(sql: string): Promise<{
+        rows: QueryDataRow[];
+        totalRows: number;
+    }>;
+    runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
+    abstract runSQLStream(sql: string, _options: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
+    runSQLBlockAndFetchResultSchema(sqlBlock: SQLBlock): Promise<{
+        data: MalloyQueryData;
+        schema: StructDef;
+    }>;
+    private getSQLBlockSchema;
+    /**
+     * Split's a structs columns declaration into individual columns
+     * to be fed back into fillStructDefFromTypeMap(). Handles commas
+     * within nested STRUCT() declarations.
+     *
+     * (https://github.com/looker-open-source/malloy/issues/635)
+     *
+     * @param s struct's column declaration
+     * @returns Array of column type declarations
+     */
+    private splitColumns;
+    private stringToTypeMap;
+    private fillStructDefFromTypeMap;
+    private schemaFromQuery;
+    fetchSchemaForSQLBlocks(sqlRefs: SQLBlock[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    fetchSchemaForTables(tables: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    private getTableSchema;
+    canFetchSchemaAndRunSimultaneously(): this is FetchSchemaAndRunSimultaneously;
+    canStream(): this is StreamingConnection;
+    canFetchSchemaAndRunStreamSimultaneously(): this is FetchSchemaAndRunStreamSimultaneously;
+    test(): Promise<void>;
+    protected abstract createHash(sqlCommand: string): Promise<string>;
+    manifestTemporaryTable(sqlCommand: string): Promise<string>;
+}

package/dist/duckdb_common.js

@@ -0,0 +1,290 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBCommon = void 0;
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+const malloy_1 = require("@malloydata/malloy");
+const duckDBToMalloyTypes = {
+    BIGINT: "number",
+    DOUBLE: "number",
+    VARCHAR: "string",
+    DATE: "date",
+    TIMESTAMP: "timestamp",
+    TIME: "string",
+    DECIMAL: "number",
+    BOOLEAN: "boolean",
+    INTEGER: "number",
+};
+class DuckDBCommon {
+    constructor(queryOptions) {
+        this.queryOptions = queryOptions;
+        this.name = "duckdb_common";
+    }
+    get dialectName() {
+        return "duckdb";
+    }
+    readQueryOptions() {
+        const options = DuckDBCommon.DEFAULT_QUERY_OPTIONS;
+        if (this.queryOptions) {
+            if (this.queryOptions instanceof Function) {
+                return { ...options, ...this.queryOptions() };
+            }
+            else {
+                return { ...options, ...this.queryOptions };
+            }
+        }
+        else {
+            return options;
+        }
+    }
+    isPool() {
+        return false;
+    }
+    canPersist() {
+        return true;
+    }
+    async runRawSQL(sql) {
+        await this.setup();
+        return this.runDuckDBQuery(sql);
+    }
+    async runSQL(sql, options = {}) {
+        var _a;
+        const defaultOptions = this.readQueryOptions();
+        const rowLimit = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : defaultOptions.rowLimit;
+        const statements = sql.split("-- hack: split on this");
+        while (statements.length > 1) {
+            await this.runRawSQL(statements[0]);
+            statements.shift();
+        }
+        const retVal = await this.runRawSQL(statements[0]);
+        let result = retVal.rows;
+        if (result.length > rowLimit) {
+            result = result.slice(0, rowLimit);
+        }
+        return { rows: result, totalRows: result.length };
+    }
+    async runSQLBlockAndFetchResultSchema(sqlBlock) {
+        const data = await this.runSQL(sqlBlock.select);
+        const schema = (await this.fetchSchemaForSQLBlocks([sqlBlock])).schemas[sqlBlock.name];
+        return { data, schema };
+    }
+    async getSQLBlockSchema(sqlRef) {
+        const structDef = {
+            type: "struct",
+            dialect: "duckdb",
+            name: sqlRef.name,
+            structSource: {
+                type: "sql",
+                method: "subquery",
+                sqlBlock: sqlRef,
+            },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.select})`, structDef);
+        return structDef;
+    }
+    /**
+     * Split's a structs columns declaration into individual columns
+     * to be fed back into fillStructDefFromTypeMap(). Handles commas
+     * within nested STRUCT() declarations.
+     *
+     * (https://github.com/looker-open-source/malloy/issues/635)
+     *
+     * @param s struct's column declaration
+     * @returns Array of column type declarations
+     */
+    splitColumns(s) {
+        const columns = [];
+        let parens = 0;
+        let column = "";
+        let eatSpaces = true;
+        for (let idx = 0; idx < s.length; idx++) {
+            const c = s.charAt(idx);
+            if (eatSpaces && c === " ") {
+                // Eat space
+            }
+            else {
+                eatSpaces = false;
+                if (!parens && c === ",") {
+                    columns.push(column);
+                    column = "";
+                    eatSpaces = true;
+                }
+                else {
+                    column += c;
+                }
+                if (c === "(") {
+                    parens += 1;
+                }
+                else if (c === ")") {
+                    parens -= 1;
+                }
+            }
+        }
+        columns.push(column);
+        return columns;
+    }
+    stringToTypeMap(s) {
+        const ret = {};
+        const columns = this.splitColumns(s);
+        for (const c of columns) {
+            //const [name, type] = c.split(" ", 1);
+            const columnMatch = c.match(/^(?<name>[^\s]+) (?<type>.*)$/);
+            if (columnMatch && columnMatch.groups) {
+                ret[columnMatch.groups["name"]] = columnMatch.groups["type"];
+            }
+            else {
+                throw new Error(`Badly form Structure definition ${s}`);
+            }
+        }
+        return ret;
+    }
+    fillStructDefFromTypeMap(structDef, typeMap) {
+        for (const name in typeMap) {
+            let duckDBType = typeMap[name];
+            // Remove DECIMAL(x,y) precision to simplify lookup
+            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, "DECIMAL");
+            let malloyType = duckDBToMalloyTypes[duckDBType];
+            const arrayMatch = duckDBType.match(/(?<duckDBType>.*)\[\]$/);
+            if (arrayMatch && arrayMatch.groups) {
+                duckDBType = arrayMatch.groups["duckDBType"];
+            }
+            const structMatch = duckDBType.match(/^STRUCT\((?<fields>.*)\)$/);
+            if (structMatch && structMatch.groups) {
+                const newTypeMap = this.stringToTypeMap(structMatch.groups["fields"]);
+                const innerStructDef = {
+                    type: "struct",
+                    name,
+                    dialect: this.dialectName,
+                    structSource: { type: arrayMatch ? "nested" : "inline" },
+                    structRelationship: {
+                        type: arrayMatch ? "nested" : "inline",
+                        field: name,
+                        isArray: false,
+                    },
+                    fields: [],
+                };
+                this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
+                structDef.fields.push(innerStructDef);
+            }
+            else {
+                if (arrayMatch) {
+                    malloyType = duckDBToMalloyTypes[duckDBType];
+                    const innerStructDef = {
+                        type: "struct",
+                        name,
+                        dialect: this.dialectName,
+                        structSource: { type: "nested" },
+                        structRelationship: { type: "nested", field: name, isArray: true },
+                        fields: [{ type: malloyType, name: "value" }],
+                    };
+                    structDef.fields.push(innerStructDef);
+                }
+                else {
+                    if (malloyType !== undefined) {
+                        structDef.fields.push({
+                            type: malloyType,
+                            name,
+                        });
+                    }
+                    else {
+                        throw new Error(`unknown duckdb type ${duckDBType}`);
+                    }
+                }
+            }
+        }
+    }
+    async schemaFromQuery(infoQuery, structDef) {
+        const typeMap = {};
+        const result = await this.runRawSQL(infoQuery);
+        for (const row of result.rows) {
+            typeMap[row["column_name"]] = row["column_type"];
+        }
+        this.fillStructDefFromTypeMap(structDef, typeMap);
+    }
+    async fetchSchemaForSQLBlocks(sqlRefs) {
+        const schemas = {};
+        const errors = {};
+        for (const sqlRef of sqlRefs) {
+            try {
+                schemas[sqlRef.name] = await this.getSQLBlockSchema(sqlRef);
+            }
+            catch (error) {
+                errors[sqlRef.name] = error;
+            }
+        }
+        return { schemas, errors };
+    }
+    async fetchSchemaForTables(tables) {
+        const schemas = {};
+        const errors = {};
+        for (const tableURL of tables) {
+            try {
+                schemas[tableURL] = await this.getTableSchema(tableURL);
+            }
+            catch (error) {
+                errors[tableURL] = error.toString();
+            }
+        }
+        return { schemas, errors };
+    }
+    async getTableSchema(tableURL) {
+        const { tablePath } = (0, malloy_1.parseTableURI)(tableURL);
+        const structDef = {
+            type: "struct",
+            name: tablePath,
+            dialect: "duckdb",
+            structSource: { type: "table", tablePath },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        const quotedTablePath = tablePath.match(/\//)
+            ? `'${tablePath}'`
+            : tablePath;
+        const infoQuery = `DESCRIBE SELECT * FROM ${quotedTablePath}`;
+        await this.schemaFromQuery(infoQuery, structDef);
+        return structDef;
+    }
+    canFetchSchemaAndRunSimultaneously() {
+        return false;
+    }
+    canStream() {
+        return true;
+    }
+    canFetchSchemaAndRunStreamSimultaneously() {
+        return false;
+    }
+    async test() {
+        await this.runRawSQL("SELECT 1");
+    }
+    async manifestTemporaryTable(sqlCommand) {
+        const hash = await this.createHash(sqlCommand);
+        const tableName = `tt${hash}`;
+        const cmd = `CREATE TEMPORARY TABLE IF NOT EXISTS ${tableName} AS (${sqlCommand});`;
+        // console.log(cmd);
+        await this.runRawSQL(cmd);
+        return tableName;
+    }
+}
+exports.DuckDBCommon = DuckDBCommon;
+DuckDBCommon.DEFAULT_QUERY_OPTIONS = {
+    rowLimit: 10,
+};
+//# sourceMappingURL=duckdb_common.js.map

package/dist/duckdb_connection.d.ts

@@ -1,60 +1,19 @@
 /// <reference path="../src/duckdb.d.ts" />
-import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, SQLBlock, StructDef, QueryDataRow } from "@malloydata/malloy";
-import { FetchSchemaAndRunSimultaneously, FetchSchemaAndRunStreamSimultaneously, StreamingConnection } from "@malloydata/malloy/src/runtime_types";
+import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
 import { Database, Row } from "duckdb";
-import { RunSQLOptions } from "@malloydata/malloy/src/malloy";
-export declare class DuckDBConnection implements Connection, PersistSQLResults, StreamingConnection {
+import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
+export declare class DuckDBConnection extends DuckDBCommon {
     readonly name: string;
     private workingDirectory;
     protected connection: import("duckdb").Connection;
     protected database: Database;
     protected isSetup: boolean;
-    constructor(name: string, databasePath?: string, workingDirectory?: string);
-    get dialectName(): string;
-    isPool(): this is PooledConnection;
-    canPersist(): this is PersistSQLResults;
+    constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
     protected setup(): Promise<void>;
     protected runDuckDBQuery(sql: string): Promise<{
         rows: Row[];
         totalRows: number;
     }>;
-    runRawSQL(sql: string): Promise<{
-        rows: Row[];
-        totalRows: number;
-    }>;
-    runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
     runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
-    runSQLBlockAndFetchResultSchema(sqlBlock: SQLBlock): Promise<{
-        data: MalloyQueryData;
-        schema: StructDef;
-    }>;
-    private getSQLBlockSchema;
-    /**
-     * Split's a structs columns declaration into individual columns
-     * to be fed back into fillStructDefFromTypeMap(). Handles commas
-     * within nested STRUCT() declarations.
-     *
-     * (https://github.com/looker-open-source/malloy/issues/635)
-     *
-     * @param s struct's column declaration
-     * @returns Array of column type declarations
-     */
-    private splitColumns;
-    private stringToTypeMap;
-    private fillStructDefFromTypeMap;
-    private schemaFromQuery;
-    fetchSchemaForSQLBlocks(sqlRefs: SQLBlock[]): Promise<{
-        schemas: Record<string, StructDef>;
-        errors: Record<string, string>;
-    }>;
-    fetchSchemaForTables(tables: string[]): Promise<{
-        schemas: Record<string, StructDef>;
-        errors: Record<string, string>;
-    }>;
-    private getTableSchema;
-    canFetchSchemaAndRunSimultaneously(): this is FetchSchemaAndRunSimultaneously;
-    canStream(): this is StreamingConnection;
-    canFetchSchemaAndRunStreamSimultaneously(): this is FetchSchemaAndRunStreamSimultaneously;
-    test(): Promise<void>;
-    manifestTemporaryTable(sqlCommand: string): Promise<string>;
+    createHash(sqlCommand: string): Promise<string>;
 }

package/dist/duckdb_connection.js

@@ -1,4 +1,16 @@
 "use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
     var desc = Object.getOwnPropertyDescriptor(m, k);
@@ -24,34 +36,12 @@ var __importStar = (this && this.__importStar) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DuckDBConnection = void 0;
-/*
- * Copyright 2021 Google LLC
- *
- * This program is free software; you can redistribute it and/or
- * modify it under the terms of the GNU General Public License
- * version 2 as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU General Public License for more details.
- */
 const crypto = __importStar(require("crypto"));
-const malloy_1 = require("@malloydata/malloy");
+const duckdb_common_1 = require("./duckdb_common");
 const duckdb_1 = require("duckdb");
-const duckDBToMalloyTypes = {
-    BIGINT: "number",
-    DOUBLE: "number",
-    VARCHAR: "string",
-    DATE: "date",
-    TIMESTAMP: "timestamp",
-    TIME: "string",
-    DECIMAL: "number",
-    BOOLEAN: "boolean",
-    INTEGER: "number",
-};
-class DuckDBConnection {
-    constructor(name, databasePath = "test/data/duckdb/duckdb_test.db", workingDirectory = "/") {
+class DuckDBConnection extends duckdb_common_1.DuckDBCommon {
+    constructor(name, databasePath = "test/data/duckdb/duckdb_test.db", workingDirectory = "/", queryOptions) {
+        super(queryOptions);
         this.name = name;
         this.workingDirectory = workingDirectory;
         this.isSetup = false;
@@ -63,15 +53,6 @@ class DuckDBConnection {
         });
         this.connection = this.database.connect();
     }
-    get dialectName() {
-        return "duckdb";
-    }
-    isPool() {
-        return false;
-    }
-    canPersist() {
-        return true;
-    }
     async setup() {
         if (!this.isSetup) {
             if (this.workingDirectory) {
@@ -98,37 +79,19 @@ class DuckDBConnection {
     }
     async runDuckDBQuery(sql) {
         return new Promise((resolve, reject) => {
-            this.connection.all(sql, (err, result) => {
+            this.connection.all(sql, (err, rows) => {
                 if (err) {
                     reject(err);
                 }
-                else
+                else {
                     resolve({
-                        rows: result,
-                        totalRows: result.length,
+                        rows,
+                        totalRows: rows.length,
                     });
+                }
             });
         });
     }
-    async runRawSQL(sql) {
-        await this.setup();
-        return this.runDuckDBQuery(sql);
-    }
-    async runSQL(sql, options = {}) {
-        var _a;
-        const rowLimit = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : 10;
-        const statements = sql.split("-- hack: split on this");
-        while (statements.length > 1) {
-            await this.runRawSQL(statements[0]);
-            statements.shift();
-        }
-        const retVal = await this.runRawSQL(statements[0]);
-        let result = retVal.rows;
-        if (result.length > rowLimit) {
-            result = result.slice(0, rowLimit);
-        }
-        return { rows: result, totalRows: result.length };
-    }
     async *runSQLStream(sql, _options = {}) {
         await this.setup();
         const statements = sql.split("-- hack: split on this");
@@ -140,221 +103,8 @@ class DuckDBConnection {
             yield row;
         }
     }
-    async runSQLBlockAndFetchResultSchema(sqlBlock) {
-        const data = await this.runSQL(sqlBlock.select);
-        const schema = (await this.fetchSchemaForSQLBlocks([sqlBlock])).schemas[sqlBlock.name];
-        return { data, schema };
-    }
-    async getSQLBlockSchema(sqlRef) {
-        const structDef = {
-            type: "struct",
-            dialect: "duckdb",
-            name: sqlRef.name,
-            structSource: {
-                type: "sql",
-                method: "subquery",
-                sqlBlock: sqlRef,
-            },
-            structRelationship: {
-                type: "basetable",
-                connectionName: this.name,
-            },
-            fields: [],
-        };
-        await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.select})`, structDef);
-        return structDef;
-    }
-    /**
-     * Split's a structs columns declaration into individual columns
-     * to be fed back into fillStructDefFromTypeMap(). Handles commas
-     * within nested STRUCT() declarations.
-     *
-     * (https://github.com/looker-open-source/malloy/issues/635)
-     *
-     * @param s struct's column declaration
-     * @returns Array of column type declarations
-     */
-    splitColumns(s) {
-        const columns = [];
-        let parens = 0;
-        let column = "";
-        let eatSpaces = true;
-        for (let idx = 0; idx < s.length; idx++) {
-            const c = s.charAt(idx);
-            if (eatSpaces && c === " ") {
-                // Eat space
-            }
-            else {
-                eatSpaces = false;
-                if (!parens && c === ",") {
-                    columns.push(column);
-                    column = "";
-                    eatSpaces = true;
-                }
-                else {
-                    column += c;
-                }
-                if (c === "(") {
-                    parens += 1;
-                }
-                else if (c === ")") {
-                    parens -= 1;
-                }
-            }
-        }
-        columns.push(column);
-        return columns;
-    }
-    stringToTypeMap(s) {
-        const ret = {};
-        const columns = this.splitColumns(s);
-        for (const c of columns) {
-            //const [name, type] = c.split(" ", 1);
-            const columnMatch = c.match(/^(?<name>[^\s]+) (?<type>.*)$/);
-            if (columnMatch && columnMatch.groups) {
-                ret[columnMatch.groups["name"]] = columnMatch.groups["type"];
-            }
-            else {
-                throw new Error(`Badly form Structure definition ${s}`);
-            }
-        }
-        return ret;
-    }
-    fillStructDefFromTypeMap(structDef, typeMap) {
-        for (const name in typeMap) {
-            let duckDBType = typeMap[name];
-            // Remove DECIMAL(x,y) precision to simplify lookup
-            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, "DECIMAL");
-            let malloyType = duckDBToMalloyTypes[duckDBType];
-            const arrayMatch = duckDBType.match(/(?<duckDBType>.*)\[\]$/);
-            if (arrayMatch && arrayMatch.groups) {
-                duckDBType = arrayMatch.groups["duckDBType"];
-            }
-            const structMatch = duckDBType.match(/^STRUCT\((?<fields>.*)\)$/);
-            if (structMatch && structMatch.groups) {
-                const newTypeMap = this.stringToTypeMap(structMatch.groups["fields"]);
-                const innerStructDef = {
-                    type: "struct",
-                    name,
-                    dialect: this.dialectName,
-                    structSource: { type: arrayMatch ? "nested" : "inline" },
-                    structRelationship: {
-                        type: arrayMatch ? "nested" : "inline",
-                        field: name,
-                        isArray: false,
-                    },
-                    fields: [],
-                };
-                this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
-                structDef.fields.push(innerStructDef);
-            }
-            else {
-                if (arrayMatch) {
-                    malloyType = duckDBToMalloyTypes[duckDBType];
-                    const innerStructDef = {
-                        type: "struct",
-                        name,
-                        dialect: this.dialectName,
-                        structSource: { type: "nested" },
-                        structRelationship: { type: "nested", field: name, isArray: true },
-                        fields: [{ type: malloyType, name: "value" }],
-                    };
-                    structDef.fields.push(innerStructDef);
-                }
-                else {
-                    if (malloyType !== undefined) {
-                        structDef.fields.push({
-                            type: malloyType,
-                            name,
-                        });
-                    }
-                    else {
-                        throw new Error(`unknown duckdb type ${duckDBType}`);
-                    }
-                }
-            }
-        }
-    }
-    async schemaFromQuery(infoQuery, structDef) {
-        const typeMap = {};
-        const result = await this.runRawSQL(infoQuery);
-        for (const row of result.rows) {
-            typeMap[row["column_name"]] = row["column_type"];
-        }
-        this.fillStructDefFromTypeMap(structDef, typeMap);
-    }
-    async fetchSchemaForSQLBlocks(sqlRefs) {
-        const schemas = {};
-        const errors = {};
-        for (const sqlRef of sqlRefs) {
-            try {
-                schemas[sqlRef.name] = await this.getSQLBlockSchema(sqlRef);
-            }
-            catch (error) {
-                errors[sqlRef.name] = error;
-            }
-        }
-        return { schemas, errors };
-    }
-    async fetchSchemaForTables(tables) {
-        const schemas = {};
-        const errors = {};
-        for (const tableURL of tables) {
-            try {
-                schemas[tableURL] = await this.getTableSchema(tableURL);
-            }
-            catch (error) {
-                errors[tableURL] = error.toString();
-            }
-        }
-        return { schemas, errors };
-    }
-    async getTableSchema(tableURL) {
-        const { tablePath: tableName } = (0, malloy_1.parseTableURL)(tableURL);
-        const structDef = {
-            type: "struct",
-            name: tableName,
-            dialect: "duckdb",
-            structSource: { type: "table" },
-            structRelationship: {
-                type: "basetable",
-                connectionName: this.name,
-            },
-            fields: [],
-        };
-        // const { tablePath: tableName } = parseTableURL(tableURL);
-        // const [schema, table] = tableName.split(".");
-        // if (table === undefined) {
-        //   throw new Error("Default schema not yet supported in DuckDB");
-        // }
-        // const infoQuery = `
-        //   SELECT column_name, data_type FROM information_schema.columns
-        //   WHERE table_name = '${table}'
-        //     AND table_schema = '${schema}'
-        // `;
-        const infoQuery = `DESCRIBE SELECT * FROM ${tableName.match(/\//) ? `'${tableName}'` : tableName};`;
-        await this.schemaFromQuery(infoQuery, structDef);
-        return structDef;
-    }
-    canFetchSchemaAndRunSimultaneously() {
-        return false;
-    }
-    canStream() {
-        return true;
-    }
-    canFetchSchemaAndRunStreamSimultaneously() {
-        return false;
-    }
-    async test() {
-        await this.runRawSQL("SELECT 1");
-    }
-    async manifestTemporaryTable(sqlCommand) {
-        const hash = crypto.createHash("md5").update(sqlCommand).digest("hex");
-        const tableName = `tt${hash}`;
-        const cmd = `CREATE TEMPORARY TABLE IF NOT EXISTS ${tableName} AS (${sqlCommand});`;
-        // console.log(cmd);
-        await this.runRawSQL(cmd);
-        return tableName;
+    createHash(sqlCommand) {
+        return Promise.resolve(crypto.createHash("md5").update(sqlCommand).digest("hex"));
     }
 }
 exports.DuckDBConnection = DuckDBConnection;

package/dist/duckdb_wasm_connection.d.ts

@@ -0,0 +1,22 @@
+import { QueryDataRow, RunSQLOptions } from "@malloydata/malloy";
+import * as duckdb from "@duckdb/duckdb-wasm";
+import { DuckDBCommon, QueryOptionsReader } from "./duckdb_common";
+export declare class DuckDBWASMConnection extends DuckDBCommon {
+    readonly name: string;
+    private workingDirectory;
+    connecting: Promise<void>;
+    protected _connection: duckdb.AsyncDuckDBConnection | null;
+    protected _database: duckdb.AsyncDuckDB | null;
+    protected isSetup: boolean;
+    constructor(name: string, databasePath?: string, workingDirectory?: string, queryOptions?: QueryOptionsReader);
+    private init;
+    get connection(): duckdb.AsyncDuckDBConnection | null;
+    get database(): duckdb.AsyncDuckDB | null;
+    protected setup(): Promise<void>;
+    protected runDuckDBQuery(sql: string): Promise<{
+        rows: QueryDataRow[];
+        totalRows: number;
+    }>;
+    runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
+    protected createHash(sqlCommand: string): Promise<string>;
+}

package/dist/duckdb_wasm_connection.js

@@ -0,0 +1,174 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBWASMConnection = void 0;
+const duckdb = __importStar(require("@duckdb/duckdb-wasm"));
+const apache_arrow_1 = require("apache-arrow");
+const duckdb_common_1 = require("./duckdb_common");
+/**
+ * Arrow's toJSON() doesn't really do what I'd expect, since
+ * it still includes Arrow objects like DecimalBigNums and Vectors,
+ * so we need this fairly gross function to unwrap those.
+ *
+ * @param value Element from an Arrow StructRow.
+ * @returns Vanilla Javascript value
+ */
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+const unwrapArrow = (value) => {
+    if (value === null) {
+        return value;
+    }
+    else if (value instanceof apache_arrow_1.Vector) {
+        return [...value].map(unwrapArrow);
+    }
+    else if (value instanceof Date) {
+        return value;
+    }
+    else if (typeof value === "bigint") {
+        return Number(value);
+    }
+    else if (typeof value === "object") {
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        const obj = value;
+        // DecimalBigNums appear as Uint32Arrays, but can be identified
+        // because they have a Symbol.toPrimitive method
+        if (obj[Symbol.toPrimitive]) {
+            // There seems to be a bug in [Symbol.toPrimitive]("number") so
+            // convert to string first and then to number.
+            return Number(obj[Symbol.toPrimitive]());
+        }
+        else if (Array.isArray(value)) {
+            return value.map(unwrapArrow);
+        }
+        else {
+            // eslint-disable-next-line @typescript-eslint/no-explicit-any
+            const result = {};
+            for (const key in obj) {
+                result[key] = unwrapArrow(obj[key]);
+            }
+            return result;
+        }
+    }
+    return value;
+};
+/**
+ * Process a single Arrow result row into a Malloy QueryDataRow
+ * Unfortunately simply calling JSONParse(JSON.stringify(row)) even
+ * winds up converting DecimalBigNums to strings instead of numbers.
+ * For some reason a custom replacer only sees DecimalBigNums as
+ * strings, as well.
+ */
+const unwrapRow = (row) => {
+    return unwrapArrow(row.toJSON());
+};
+/**
+ * Process a duckedb Table into an array of Malloy QueryDataRows
+ */
+const unwrapTable = (table) => {
+    return table.toArray().map(unwrapRow);
+};
+class DuckDBWASMConnection extends duckdb_common_1.DuckDBCommon {
+    constructor(name, databasePath = "test/data/duckdb/duckdb_test.db", workingDirectory = "/", queryOptions) {
+        super(queryOptions);
+        this.name = name;
+        this.workingDirectory = workingDirectory;
+        this._connection = null;
+        this._database = null;
+        this.isSetup = false;
+        this.connecting = this.init();
+    }
+    async init() {
+        const JSDELIVR_BUNDLES = duckdb.getJsDelivrBundles();
+        // Select a bundle based on browser checks
+        const bundle = await duckdb.selectBundle(JSDELIVR_BUNDLES);
+        if (bundle.mainWorker) {
+            const workerUrl = URL.createObjectURL(new Blob([`importScripts("${bundle.mainWorker}");`], {
+                type: "text/javascript",
+            }));
+            // Instantiate the asynchronous version of DuckDB-wasm
+            const worker = new Worker(workerUrl);
+            const logger = new duckdb.ConsoleLogger();
+            this._database = new duckdb.AsyncDuckDB(logger, worker);
+            await this._database.instantiate(bundle.mainModule, bundle.pthreadWorker);
+            URL.revokeObjectURL(workerUrl);
+            this._connection = await this._database.connect();
+        }
+        else {
+            throw new Error("Unable to instantiate duckdb-wasm");
+        }
+    }
+    get connection() {
+        return this._connection;
+    }
+    get database() {
+        return this._database;
+    }
+    async setup() {
+        await this.connecting;
+    }
+    async runDuckDBQuery(sql) {
+        var _a;
+        const table = await ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.query(sql));
+        if ((table === null || table === void 0 ? void 0 : table.numRows) != null) {
+            const rows = unwrapTable(table);
+            // console.log(rows);
+            return {
+                // Normalize the data from its default proxied form
+                rows,
+                totalRows: table.numRows,
+            };
+        }
+        else {
+            throw new Error("Boom");
+        }
+    }
+    async *runSQLStream(sql, _options = {}) {
+        if (!this.connection) {
+            throw new Error("duckdb-wasm not connected");
+        }
+        await this.setup();
+        const statements = sql.split("-- hack: split on this");
+        while (statements.length > 1) {
+            await this.runDuckDBQuery(statements[0]);
+            statements.shift();
+        }
+        for await (const chunk of await this.connection.send(statements[0])) {
+            for (const row of chunk.toArray()) {
+                yield unwrapRow(row);
+            }
+        }
+    }
+    async createHash(sqlCommand) {
+        const msgUint8 = new TextEncoder().encode(sqlCommand);
+        const hashBuffer = await crypto.subtle.digest("SHA-256", msgUint8);
+        const hashArray = Array.from(new Uint8Array(hashBuffer));
+        const hashHex = hashArray
+            .map((b) => b.toString(16).padStart(2, "0"))
+            .join("");
+        return hashHex;
+    }
+}
+exports.DuckDBWASMConnection = DuckDBWASMConnection;
+//# sourceMappingURL=duckdb_wasm_connection.js.map

package/package.json

@@ -1,9 +1,14 @@
 {
   "name": "@malloydata/db-duckdb",
-  "version": "0.0.1",
+  "version": "0.0.4",
   "license": "GPL-2.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
+  "homepage": "https://github.com/looker-open-source/malloy#readme",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/looker-open-source/malloy"
+  },
   "scripts": {
     "lint": "eslint '**/*.ts{,x}'",
     "lint-fix": "eslint '**/*.ts{,x}' --fix",
@@ -12,6 +17,8 @@
     "malloyc": "ts-node ../../scripts/malloy-to-json"
   },
   "dependencies": {
-    "duckdb": "0.4.1-dev1094.0"
+    "@duckdb/duckdb-wasm": "^1.17.0",
+    "@malloydata/malloy": "^0.0.4",
+    "duckdb": "0.5.1"
   }
 }