@malloydata/db-duckdb 0.0.1

package/README.md

@@ -0,0 +1,7 @@
+# Malloy
+
+Malloy is an experimental language for describing data relationships and transformations. It is both a semantic modeling language and a querying language that runs queries against a relational database. Malloy currently connects to BigQuery and Postgres, and natively supports DuckDB. We've built a Visual Studio Code extension to facilitate building Malloy data models, querying and transforming data, and creating simple visualizations and dashboards.
+
+## This package
+
+This package facilitates using the `malloydata/malloy` library with DuckDB - see [here](https://github.com/looker-open-source/malloy/blob/main/packages/malloy/README.md) for additional information.

package/dist/duckdb_connection.d.ts

@@ -0,0 +1,60 @@
+/// <reference path="../src/duckdb.d.ts" />
+import { Connection, MalloyQueryData, PersistSQLResults, PooledConnection, SQLBlock, StructDef, QueryDataRow } from "@malloydata/malloy";
+import { FetchSchemaAndRunSimultaneously, FetchSchemaAndRunStreamSimultaneously, StreamingConnection } from "@malloydata/malloy/src/runtime_types";
+import { Database, Row } from "duckdb";
+import { RunSQLOptions } from "@malloydata/malloy/src/malloy";
+export declare class DuckDBConnection implements Connection, PersistSQLResults, StreamingConnection {
+    readonly name: string;
+    private workingDirectory;
+    protected connection: import("duckdb").Connection;
+    protected database: Database;
+    protected isSetup: boolean;
+    constructor(name: string, databasePath?: string, workingDirectory?: string);
+    get dialectName(): string;
+    isPool(): this is PooledConnection;
+    canPersist(): this is PersistSQLResults;
+    protected setup(): Promise<void>;
+    protected runDuckDBQuery(sql: string): Promise<{
+        rows: Row[];
+        totalRows: number;
+    }>;
+    runRawSQL(sql: string): Promise<{
+        rows: Row[];
+        totalRows: number;
+    }>;
+    runSQL(sql: string, options?: RunSQLOptions): Promise<MalloyQueryData>;
+    runSQLStream(sql: string, _options?: RunSQLOptions): AsyncIterableIterator<QueryDataRow>;
+    runSQLBlockAndFetchResultSchema(sqlBlock: SQLBlock): Promise<{
+        data: MalloyQueryData;
+        schema: StructDef;
+    }>;
+    private getSQLBlockSchema;
+    /**
+     * Split's a structs columns declaration into individual columns
+     * to be fed back into fillStructDefFromTypeMap(). Handles commas
+     * within nested STRUCT() declarations.
+     *
+     * (https://github.com/looker-open-source/malloy/issues/635)
+     *
+     * @param s struct's column declaration
+     * @returns Array of column type declarations
+     */
+    private splitColumns;
+    private stringToTypeMap;
+    private fillStructDefFromTypeMap;
+    private schemaFromQuery;
+    fetchSchemaForSQLBlocks(sqlRefs: SQLBlock[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    fetchSchemaForTables(tables: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    private getTableSchema;
+    canFetchSchemaAndRunSimultaneously(): this is FetchSchemaAndRunSimultaneously;
+    canStream(): this is StreamingConnection;
+    canFetchSchemaAndRunStreamSimultaneously(): this is FetchSchemaAndRunStreamSimultaneously;
+    test(): Promise<void>;
+    manifestTemporaryTable(sqlCommand: string): Promise<string>;
+}

package/dist/duckdb_connection.js

@@ -0,0 +1,361 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBConnection = void 0;
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+const crypto = __importStar(require("crypto"));
+const malloy_1 = require("@malloydata/malloy");
+const duckdb_1 = require("duckdb");
+const duckDBToMalloyTypes = {
+    BIGINT: "number",
+    DOUBLE: "number",
+    VARCHAR: "string",
+    DATE: "date",
+    TIMESTAMP: "timestamp",
+    TIME: "string",
+    DECIMAL: "number",
+    BOOLEAN: "boolean",
+    INTEGER: "number",
+};
+class DuckDBConnection {
+    constructor(name, databasePath = "test/data/duckdb/duckdb_test.db", workingDirectory = "/") {
+        this.name = name;
+        this.workingDirectory = workingDirectory;
+        this.isSetup = false;
+        this.database = new duckdb_1.Database(databasePath, duckdb_1.OPEN_READWRITE, // databasePath === ":memory:" ? OPEN_READWRITE : OPEN_READONLY,
+        (err) => {
+            if (err) {
+                return console.error(err);
+            }
+        });
+        this.connection = this.database.connect();
+    }
+    get dialectName() {
+        return "duckdb";
+    }
+    isPool() {
+        return false;
+    }
+    canPersist() {
+        return true;
+    }
+    async setup() {
+        if (!this.isSetup) {
+            if (this.workingDirectory) {
+                this.runDuckDBQuery(`SET FILE_SEARCH_PATH='${this.workingDirectory}'`);
+            }
+            // TODO: This is where we will load extensions once we figure
+            // out how to better support them.
+            // await this.runDuckDBQuery("INSTALL 'json'");
+            // await this.runDuckDBQuery("LOAD 'json'");
+            // await this.runDuckDBQuery("INSTALL 'httpfs'");
+            // await this.runDuckDBQuery("LOAD 'httpfs'");
+            //   await this.runDuckDBQuery("DROP MACRO sum_distinct");
+            //   try {
+            //     await this.runDuckDBQuery(
+            //       `
+            //       create macro sum_distinct(l) as  (
+            //         select sum(x.val) as value FROM (select unnest(l)) x
+            //       )
+            //       `
+            //     );
+            //   } catch (e) {}
+        }
+        this.isSetup = true;
+    }
+    async runDuckDBQuery(sql) {
+        return new Promise((resolve, reject) => {
+            this.connection.all(sql, (err, result) => {
+                if (err) {
+                    reject(err);
+                }
+                else
+                    resolve({
+                        rows: result,
+                        totalRows: result.length,
+                    });
+            });
+        });
+    }
+    async runRawSQL(sql) {
+        await this.setup();
+        return this.runDuckDBQuery(sql);
+    }
+    async runSQL(sql, options = {}) {
+        var _a;
+        const rowLimit = (_a = options.rowLimit) !== null && _a !== void 0 ? _a : 10;
+        const statements = sql.split("-- hack: split on this");
+        while (statements.length > 1) {
+            await this.runRawSQL(statements[0]);
+            statements.shift();
+        }
+        const retVal = await this.runRawSQL(statements[0]);
+        let result = retVal.rows;
+        if (result.length > rowLimit) {
+            result = result.slice(0, rowLimit);
+        }
+        return { rows: result, totalRows: result.length };
+    }
+    async *runSQLStream(sql, _options = {}) {
+        await this.setup();
+        const statements = sql.split("-- hack: split on this");
+        while (statements.length > 1) {
+            await this.runDuckDBQuery(statements[0]);
+            statements.shift();
+        }
+        for await (const row of this.connection.stream(statements[0])) {
+            yield row;
+        }
+    }
+    async runSQLBlockAndFetchResultSchema(sqlBlock) {
+        const data = await this.runSQL(sqlBlock.select);
+        const schema = (await this.fetchSchemaForSQLBlocks([sqlBlock])).schemas[sqlBlock.name];
+        return { data, schema };
+    }
+    async getSQLBlockSchema(sqlRef) {
+        const structDef = {
+            type: "struct",
+            dialect: "duckdb",
+            name: sqlRef.name,
+            structSource: {
+                type: "sql",
+                method: "subquery",
+                sqlBlock: sqlRef,
+            },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        await this.schemaFromQuery(`DESCRIBE SELECT * FROM (${sqlRef.select})`, structDef);
+        return structDef;
+    }
+    /**
+     * Split's a structs columns declaration into individual columns
+     * to be fed back into fillStructDefFromTypeMap(). Handles commas
+     * within nested STRUCT() declarations.
+     *
+     * (https://github.com/looker-open-source/malloy/issues/635)
+     *
+     * @param s struct's column declaration
+     * @returns Array of column type declarations
+     */
+    splitColumns(s) {
+        const columns = [];
+        let parens = 0;
+        let column = "";
+        let eatSpaces = true;
+        for (let idx = 0; idx < s.length; idx++) {
+            const c = s.charAt(idx);
+            if (eatSpaces && c === " ") {
+                // Eat space
+            }
+            else {
+                eatSpaces = false;
+                if (!parens && c === ",") {
+                    columns.push(column);
+                    column = "";
+                    eatSpaces = true;
+                }
+                else {
+                    column += c;
+                }
+                if (c === "(") {
+                    parens += 1;
+                }
+                else if (c === ")") {
+                    parens -= 1;
+                }
+            }
+        }
+        columns.push(column);
+        return columns;
+    }
+    stringToTypeMap(s) {
+        const ret = {};
+        const columns = this.splitColumns(s);
+        for (const c of columns) {
+            //const [name, type] = c.split(" ", 1);
+            const columnMatch = c.match(/^(?<name>[^\s]+) (?<type>.*)$/);
+            if (columnMatch && columnMatch.groups) {
+                ret[columnMatch.groups["name"]] = columnMatch.groups["type"];
+            }
+            else {
+                throw new Error(`Badly form Structure definition ${s}`);
+            }
+        }
+        return ret;
+    }
+    fillStructDefFromTypeMap(structDef, typeMap) {
+        for (const name in typeMap) {
+            let duckDBType = typeMap[name];
+            // Remove DECIMAL(x,y) precision to simplify lookup
+            duckDBType = duckDBType.replace(/^DECIMAL\(\d+,\d+\)/g, "DECIMAL");
+            let malloyType = duckDBToMalloyTypes[duckDBType];
+            const arrayMatch = duckDBType.match(/(?<duckDBType>.*)\[\]$/);
+            if (arrayMatch && arrayMatch.groups) {
+                duckDBType = arrayMatch.groups["duckDBType"];
+            }
+            const structMatch = duckDBType.match(/^STRUCT\((?<fields>.*)\)$/);
+            if (structMatch && structMatch.groups) {
+                const newTypeMap = this.stringToTypeMap(structMatch.groups["fields"]);
+                const innerStructDef = {
+                    type: "struct",
+                    name,
+                    dialect: this.dialectName,
+                    structSource: { type: arrayMatch ? "nested" : "inline" },
+                    structRelationship: {
+                        type: arrayMatch ? "nested" : "inline",
+                        field: name,
+                        isArray: false,
+                    },
+                    fields: [],
+                };
+                this.fillStructDefFromTypeMap(innerStructDef, newTypeMap);
+                structDef.fields.push(innerStructDef);
+            }
+            else {
+                if (arrayMatch) {
+                    malloyType = duckDBToMalloyTypes[duckDBType];
+                    const innerStructDef = {
+                        type: "struct",
+                        name,
+                        dialect: this.dialectName,
+                        structSource: { type: "nested" },
+                        structRelationship: { type: "nested", field: name, isArray: true },
+                        fields: [{ type: malloyType, name: "value" }],
+                    };
+                    structDef.fields.push(innerStructDef);
+                }
+                else {
+                    if (malloyType !== undefined) {
+                        structDef.fields.push({
+                            type: malloyType,
+                            name,
+                        });
+                    }
+                    else {
+                        throw new Error(`unknown duckdb type ${duckDBType}`);
+                    }
+                }
+            }
+        }
+    }
+    async schemaFromQuery(infoQuery, structDef) {
+        const typeMap = {};
+        const result = await this.runRawSQL(infoQuery);
+        for (const row of result.rows) {
+            typeMap[row["column_name"]] = row["column_type"];
+        }
+        this.fillStructDefFromTypeMap(structDef, typeMap);
+    }
+    async fetchSchemaForSQLBlocks(sqlRefs) {
+        const schemas = {};
+        const errors = {};
+        for (const sqlRef of sqlRefs) {
+            try {
+                schemas[sqlRef.name] = await this.getSQLBlockSchema(sqlRef);
+            }
+            catch (error) {
+                errors[sqlRef.name] = error;
+            }
+        }
+        return { schemas, errors };
+    }
+    async fetchSchemaForTables(tables) {
+        const schemas = {};
+        const errors = {};
+        for (const tableURL of tables) {
+            try {
+                schemas[tableURL] = await this.getTableSchema(tableURL);
+            }
+            catch (error) {
+                errors[tableURL] = error.toString();
+            }
+        }
+        return { schemas, errors };
+    }
+    async getTableSchema(tableURL) {
+        const { tablePath: tableName } = (0, malloy_1.parseTableURL)(tableURL);
+        const structDef = {
+            type: "struct",
+            name: tableName,
+            dialect: "duckdb",
+            structSource: { type: "table" },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        // const { tablePath: tableName } = parseTableURL(tableURL);
+        // const [schema, table] = tableName.split(".");
+        // if (table === undefined) {
+        //   throw new Error("Default schema not yet supported in DuckDB");
+        // }
+        // const infoQuery = `
+        //   SELECT column_name, data_type FROM information_schema.columns
+        //   WHERE table_name = '${table}'
+        //     AND table_schema = '${schema}'
+        // `;
+        const infoQuery = `DESCRIBE SELECT * FROM ${tableName.match(/\//) ? `'${tableName}'` : tableName};`;
+        await this.schemaFromQuery(infoQuery, structDef);
+        return structDef;
+    }
+    canFetchSchemaAndRunSimultaneously() {
+        return false;
+    }
+    canStream() {
+        return true;
+    }
+    canFetchSchemaAndRunStreamSimultaneously() {
+        return false;
+    }
+    async test() {
+        await this.runRawSQL("SELECT 1");
+    }
+    async manifestTemporaryTable(sqlCommand) {
+        const hash = crypto.createHash("md5").update(sqlCommand).digest("hex");
+        const tableName = `tt${hash}`;
+        const cmd = `CREATE TEMPORARY TABLE IF NOT EXISTS ${tableName} AS (${sqlCommand});`;
+        // console.log(cmd);
+        await this.runRawSQL(cmd);
+        return tableName;
+    }
+}
+exports.DuckDBConnection = DuckDBConnection;
+//# sourceMappingURL=duckdb_connection.js.map

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { DuckDBConnection } from "./duckdb_connection";

package/dist/index.js

@@ -0,0 +1,18 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DuckDBConnection = void 0;
+var duckdb_connection_1 = require("./duckdb_connection");
+Object.defineProperty(exports, "DuckDBConnection", { enumerable: true, get: function () { return duckdb_connection_1.DuckDBConnection; } });
+//# sourceMappingURL=index.js.map

package/package.json

@@ -0,0 +1,17 @@
+{
+  "name": "@malloydata/db-duckdb",
+  "version": "0.0.1",
+  "license": "GPL-2.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "lint": "eslint '**/*.ts{,x}'",
+    "lint-fix": "eslint '**/*.ts{,x}' --fix",
+    "test": "jest --config=../../jest.config.js",
+    "build": "tsc --build",
+    "malloyc": "ts-node ../../scripts/malloy-to-json"
+  },
+  "dependencies": {
+    "duckdb": "0.4.1-dev1094.0"
+  }
+}