@malloydata/db-postgres 0.0.1

package/README.md

@@ -0,0 +1,7 @@
+# Malloy
+
+Malloy is an experimental language for describing data relationships and transformations. It is both a semantic modeling language and a querying language that runs queries against a relational database. Malloy currently connects to BigQuery and Postgres, and natively supports DuckDB. We've built a Visual Studio Code extension to facilitate building Malloy data models, querying and transforming data, and creating simple visualizations and dashboards.
+
+## This package
+
+This package facilitates using the `malloydata/malloy` library with Postgres - see [here](https://github.com/looker-open-source/malloy/blob/main/packages/malloy/README.md) for additional information.

package/dist/index.d.ts

@@ -0,0 +1 @@
+export { PostgresConnection, PooledPostgresConnection, } from "./postgres_connection";

package/dist/index.js

@@ -0,0 +1,19 @@
+"use strict";
+/*
+ * Copyright 2021 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PooledPostgresConnection = exports.PostgresConnection = void 0;
+var postgres_connection_1 = require("./postgres_connection");
+Object.defineProperty(exports, "PostgresConnection", { enumerable: true, get: function () { return postgres_connection_1.PostgresConnection; } });
+Object.defineProperty(exports, "PooledPostgresConnection", { enumerable: true, get: function () { return postgres_connection_1.PooledPostgresConnection; } });
+//# sourceMappingURL=index.js.map

package/dist/postgres_connection.d.ts

@@ -0,0 +1,64 @@
+import { StructDef, MalloyQueryData, QueryData, PooledConnection, SQLBlock, Connection, QueryDataRow } from "@malloydata/malloy";
+import { FetchSchemaAndRunSimultaneously, FetchSchemaAndRunStreamSimultaneously, PersistSQLResults, StreamingConnection } from "@malloydata/malloy/src/runtime_types";
+import { Client } from "pg";
+import { RunSQLOptions } from "@malloydata/malloy/src/malloy";
+interface PostgresQueryConfiguration {
+    rowLimit?: number;
+}
+declare type PostgresQueryConfigurationReader = PostgresQueryConfiguration | (() => PostgresQueryConfiguration) | (() => Promise<PostgresQueryConfiguration>);
+interface PostgresConnectionConfiguration {
+    host?: string;
+    port?: number;
+    username?: string;
+    password?: string;
+    databaseName?: string;
+}
+declare type PostgresConnectionConfigurationReader = PostgresConnectionConfiguration | (() => Promise<PostgresConnectionConfiguration>);
+export declare class PostgresConnection implements Connection, StreamingConnection, PersistSQLResults {
+    private schemaCache;
+    private sqlSchemaCache;
+    private queryConfigReader;
+    private configReader;
+    readonly name: string;
+    constructor(name: string, queryConfigReader?: PostgresQueryConfigurationReader, configReader?: PostgresConnectionConfigurationReader);
+    private readQueryConfig;
+    private readConfig;
+    get dialectName(): string;
+    isPool(): this is PooledConnection;
+    canPersist(): this is PersistSQLResults;
+    canFetchSchemaAndRunSimultaneously(): this is FetchSchemaAndRunSimultaneously;
+    canFetchSchemaAndRunStreamSimultaneously(): this is FetchSchemaAndRunStreamSimultaneously;
+    canStream(): this is StreamingConnection;
+    fetchSchemaForTables(missing: string[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    fetchSchemaForSQLBlocks(sqlRefs: SQLBlock[]): Promise<{
+        schemas: Record<string, StructDef>;
+        errors: Record<string, string>;
+    }>;
+    protected getClient(): Promise<Client>;
+    protected runPostgresQuery(sqlCommand: string, _pageSize: number, _rowIndex: number, deJSON: boolean): Promise<MalloyQueryData>;
+    private getSQLBlockSchema;
+    private schemaFromQuery;
+    private getTableSchema;
+    executeSQLRaw(query: string): Promise<QueryData>;
+    test(): Promise<void>;
+    runSQL(sql: string, { rowLimit }?: RunSQLOptions, rowIndex?: number): Promise<MalloyQueryData>;
+    runSQLStream(sqlCommand: string, options?: {
+        rowLimit?: number;
+    }): AsyncIterableIterator<QueryDataRow>;
+    manifestTemporaryTable(sqlCommand: string): Promise<string>;
+}
+export declare class PooledPostgresConnection extends PostgresConnection implements PooledConnection {
+    private pool;
+    constructor(name: string);
+    isPool(): true;
+    drain(): Promise<void>;
+    protected runPostgresQuery(sqlCommand: string, _pageSize: number, _rowIndex: number, deJSON: boolean): Promise<MalloyQueryData>;
+    private getClientFromPool;
+    runSQLStream(sqlCommand: string, options?: {
+        rowLimit?: number;
+    }): AsyncIterableIterator<QueryDataRow>;
+}
+export {};

package/dist/postgres_connection.js

@@ -0,0 +1,382 @@
+"use strict";
+/*
+ * Copyright 2022 Google LLC
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * version 2 as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PooledPostgresConnection = exports.PostgresConnection = void 0;
+// LTNOTE: we need this extension to be installed to correctly index
+//  postgres data...  We should probably do this on connection creation...
+//
+//     create extension if not exists tsm_system_rows
+//
+const crypto = __importStar(require("crypto"));
+const malloy_1 = require("@malloydata/malloy");
+const pg_1 = require("pg");
+const pg_query_stream_1 = __importDefault(require("pg-query-stream"));
+const postgresToMalloyTypes = {
+    "character varying": "string",
+    name: "string",
+    text: "string",
+    date: "date",
+    integer: "number",
+    bigint: "number",
+    "double precision": "number",
+    "timestamp without time zone": "timestamp",
+    oid: "string",
+    boolean: "boolean",
+    // ARRAY: "string",
+    "timestamp with time zone": "timestamp",
+    timestamp: "timestamp",
+    '"char"': "string",
+    character: "string",
+    smallint: "number",
+    xid: "string",
+    real: "number",
+    interval: "string",
+    inet: "string",
+    regtype: "string",
+    numeric: "number",
+    bytea: "string",
+    pg_ndistinct: "number",
+};
+const DEFAULT_PAGE_SIZE = 1000;
+const SCHEMA_PAGE_SIZE = 1000;
+class PostgresConnection {
+    constructor(name, queryConfigReader = {}, configReader = {}) {
+        this.schemaCache = new Map();
+        this.sqlSchemaCache = new Map();
+        this.queryConfigReader = queryConfigReader;
+        this.configReader = configReader;
+        this.name = name;
+    }
+    async readQueryConfig() {
+        if (this.queryConfigReader instanceof Function) {
+            return this.queryConfigReader();
+        }
+        else {
+            return this.queryConfigReader;
+        }
+    }
+    async readConfig() {
+        if (this.configReader instanceof Function) {
+            return this.configReader();
+        }
+        else {
+            return this.configReader;
+        }
+    }
+    get dialectName() {
+        return "postgres";
+    }
+    isPool() {
+        return false;
+    }
+    canPersist() {
+        return true;
+    }
+    canFetchSchemaAndRunSimultaneously() {
+        // TODO feature-sql-block Implement FetchSchemaAndRunSimultaneously
+        return false;
+    }
+    canFetchSchemaAndRunStreamSimultaneously() {
+        return false;
+    }
+    canStream() {
+        return true;
+    }
+    async fetchSchemaForTables(missing) {
+        const schemas = {};
+        const errors = {};
+        for (const tableURL of missing) {
+            let inCache = this.schemaCache.get(tableURL);
+            if (!inCache) {
+                try {
+                    inCache = {
+                        schema: await this.getTableSchema(tableURL),
+                    };
+                    this.schemaCache.set(tableURL, inCache);
+                }
+                catch (error) {
+                    inCache = { error: error.message };
+                }
+            }
+            if (inCache.schema !== undefined) {
+                schemas[tableURL] = inCache.schema;
+            }
+            else {
+                errors[tableURL] = inCache.error;
+            }
+        }
+        return { schemas, errors };
+    }
+    async fetchSchemaForSQLBlocks(sqlRefs) {
+        const schemas = {};
+        const errors = {};
+        for (const sqlRef of sqlRefs) {
+            const key = sqlRef.name;
+            let inCache = this.sqlSchemaCache.get(key);
+            if (!inCache) {
+                try {
+                    inCache = {
+                        schema: await this.getSQLBlockSchema(sqlRef),
+                    };
+                    this.schemaCache.set(key, inCache);
+                }
+                catch (error) {
+                    inCache = { error: error.message };
+                }
+            }
+            if (inCache.schema !== undefined) {
+                schemas[key] = inCache.schema;
+            }
+            else {
+                errors[key] = inCache.error;
+            }
+        }
+        return { schemas, errors };
+    }
+    async getClient() {
+        const config = await this.readConfig();
+        return new pg_1.Client({
+            user: config.username,
+            password: config.password,
+            database: config.databaseName,
+            port: config.port,
+            host: config.host,
+        });
+    }
+    async runPostgresQuery(sqlCommand, _pageSize, _rowIndex, deJSON) {
+        const client = await this.getClient();
+        await client.connect();
+        let result = await client.query(sqlCommand);
+        if (Array.isArray(result)) {
+            result = result.pop();
+        }
+        if (deJSON) {
+            for (let i = 0; i < result.rows.length; i++) {
+                result.rows[i] = result.rows[i].row;
+            }
+        }
+        await client.end();
+        return { rows: result.rows, totalRows: result.rows.length };
+    }
+    async getSQLBlockSchema(sqlRef) {
+        const structDef = {
+            type: "struct",
+            dialect: "postgres",
+            name: sqlRef.name,
+            structSource: {
+                type: "sql",
+                method: "subquery",
+                sqlBlock: sqlRef,
+            },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        // TODO -- Should be a uuid
+        const tempTableName = `malloy${Math.floor(Math.random() * 10000000)}`;
+        const infoQuery = `
+      drop table if exists ${tempTableName};
+      create temp table ${tempTableName} as SELECT * FROM (
+        ${sqlRef.select}
+      ) as x where false;
+      SELECT column_name, c.data_type, e.data_type as element_type
+      FROM information_schema.columns c LEFT JOIN information_schema.element_types e
+        ON ((c.table_catalog, c.table_schema, c.table_name, 'TABLE', c.dtd_identifier)
+          = (e.object_catalog, e.object_schema, e.object_name, e.object_type, e.collection_type_identifier))
+      where table_name='${tempTableName}';
+    `;
+        await this.schemaFromQuery(infoQuery, structDef);
+        return structDef;
+    }
+    async schemaFromQuery(infoQuery, structDef) {
+        const result = await this.runPostgresQuery(infoQuery, SCHEMA_PAGE_SIZE, 0, false);
+        for (const row of result.rows) {
+            const postgresDataType = row["data_type"];
+            let s = structDef;
+            let malloyType = postgresToMalloyTypes[postgresDataType];
+            let name = row["column_name"];
+            if (postgresDataType === "ARRAY") {
+                malloyType = postgresToMalloyTypes[row["element_type"]];
+                s = {
+                    type: "struct",
+                    name: row["column_name"],
+                    dialect: this.dialectName,
+                    structRelationship: { type: "nested", field: name, isArray: true },
+                    structSource: { type: "nested" },
+                    fields: [],
+                };
+                structDef.fields.push(s);
+                name = "value";
+            }
+            if (malloyType !== undefined) {
+                s.fields.push({
+                    type: malloyType,
+                    name,
+                });
+            }
+            else {
+                throw new Error(`unknown postgres type ${postgresDataType}`);
+            }
+        }
+    }
+    async getTableSchema(tableURL) {
+        const structDef = {
+            type: "struct",
+            name: tableURL,
+            dialect: "postgres",
+            structSource: { type: "table" },
+            structRelationship: {
+                type: "basetable",
+                connectionName: this.name,
+            },
+            fields: [],
+        };
+        const { tablePath: tableName } = (0, malloy_1.parseTableURL)(tableURL);
+        const [schema, table] = tableName.split(".");
+        if (table === undefined) {
+            throw new Error("Default schema not yet supported in Postgres");
+        }
+        const infoQuery = `
+      SELECT column_name, c.data_type, e.data_type as element_type
+      FROM information_schema.columns c LEFT JOIN information_schema.element_types e
+        ON ((c.table_catalog, c.table_schema, c.table_name, 'TABLE', c.dtd_identifier)
+          = (e.object_catalog, e.object_schema, e.object_name, e.object_type, e.collection_type_identifier))
+        WHERE table_name = '${table}'
+          AND table_schema = '${schema}'
+    `;
+        await this.schemaFromQuery(infoQuery, structDef);
+        return structDef;
+    }
+    async executeSQLRaw(query) {
+        const config = await this.readQueryConfig();
+        const queryData = await this.runPostgresQuery(query, config.rowLimit || DEFAULT_PAGE_SIZE, 0, false);
+        return queryData.rows;
+    }
+    async test() {
+        await this.executeSQLRaw("SELECT 1");
+    }
+    async runSQL(sql, { rowLimit } = {}, rowIndex = 0) {
+        var _a;
+        const config = await this.readQueryConfig();
+        return await this.runPostgresQuery(sql, (_a = rowLimit !== null && rowLimit !== void 0 ? rowLimit : config.rowLimit) !== null && _a !== void 0 ? _a : DEFAULT_PAGE_SIZE, rowIndex, true);
+    }
+    async *runSQLStream(sqlCommand, options) {
+        const query = new pg_query_stream_1.default(sqlCommand);
+        const client = await this.getClient();
+        client.connect();
+        const rowStream = client.query(query);
+        let index = 0;
+        for await (const row of rowStream) {
+            yield row.row;
+            index += 1;
+            if ((options === null || options === void 0 ? void 0 : options.rowLimit) !== undefined && index >= options.rowLimit) {
+                query.destroy();
+                break;
+            }
+        }
+        await client.end();
+    }
+    async manifestTemporaryTable(sqlCommand) {
+        const hash = crypto.createHash("md5").update(sqlCommand).digest("hex");
+        const tableName = `tt${hash}`;
+        const cmd = `CREATE TEMPORARY TABLE IF NOT EXISTS ${tableName} AS (${sqlCommand});`;
+        // console.log(cmd);
+        await this.runPostgresQuery(cmd, 1000, 0, false);
+        return tableName;
+    }
+}
+exports.PostgresConnection = PostgresConnection;
+class PooledPostgresConnection extends PostgresConnection {
+    constructor(name) {
+        super(name);
+        this.pool = new pg_1.Pool();
+    }
+    isPool() {
+        return true;
+    }
+    async drain() {
+        await this.pool.end();
+    }
+    async runPostgresQuery(sqlCommand, _pageSize, _rowIndex, deJSON) {
+        let result = await this.pool.query(sqlCommand);
+        if (Array.isArray(result)) {
+            result = result.pop();
+        }
+        if (deJSON) {
+            for (let i = 0; i < result.rows.length; i++) {
+                result.rows[i] = result.rows[i].row;
+            }
+        }
+        return { rows: result.rows, totalRows: result.rows.length };
+    }
+    async getClientFromPool() {
+        return await new Promise((resolve, reject) => this.pool.connect((error, client, releaseClient) => {
+            if (error) {
+                reject(error);
+            }
+            else {
+                resolve([client, releaseClient]);
+            }
+        }));
+    }
+    async *runSQLStream(sqlCommand, options) {
+        const query = new pg_query_stream_1.default(sqlCommand);
+        let index = 0;
+        // This is a strange hack... `this.pool.query(query)` seems to return the wrong
+        // type. Because `query` is a `QueryStream`, the result is supposed to be a
+        // `QueryStream` as well, but it's not. So instead, we get a client and call
+        // `client.query(query)`, which does what it's supposed to.
+        const [client, releaseClient] = await this.getClientFromPool();
+        const resultStream = client.query(query);
+        for await (const row of resultStream) {
+            yield row.row;
+            index += 1;
+            if ((options === null || options === void 0 ? void 0 : options.rowLimit) !== undefined && index >= options.rowLimit) {
+                query.destroy();
+                break;
+            }
+        }
+        releaseClient();
+    }
+}
+exports.PooledPostgresConnection = PooledPostgresConnection;
+//# sourceMappingURL=postgres_connection.js.map

package/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "@malloydata/db-postgres",
+  "version": "0.0.1",
+  "license": "GPL-2.0",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "scripts": {
+    "lint": "eslint '**/*.ts{,x}'",
+    "lint-fix": "eslint '**/*.ts{,x}' --fix",
+    "test": "jest --config=../../jest.config.js",
+    "build": "tsc --build",
+    "malloyc": "ts-node ../../scripts/malloy-to-json"
+  },
+  "dependencies": {
+    "@malloydata/malloy": "^0.0.1",
+    "@types/pg": "^8.6.1",
+    "pg": "^8.7.1",
+    "pg-query-stream": "4.2.3"
+  }
+}