@malloydata/db-postgres 0.0.195-dev241003204819 → 0.0.195-dev241007154000

package/dist/postgres.spec.js

@@ -45,34 +45,23 @@ describe('PostgresConnection', () => {
     beforeEach(async () => {
         getTableSchema = jest
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .spyOn(postgres_connection_1.PostgresConnection.prototype, 'getTableSchema')
+            .spyOn(postgres_connection_1.PostgresConnection.prototype, 'fetchTableSchema')
             .mockResolvedValue({
-            type: 'struct',
+            type: 'table',
             dialect: 'postgres',
             name: 'name',
-            structSource: { type: 'table', tablePath: 'test' },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: 'postgres',
-            },
-            fields: [],
+            tablePath: 'test',
+            connection: 'postgres',
         });
         getSQLBlockSchema = jest
             // eslint-disable-next-line @typescript-eslint/no-explicit-any
-            .spyOn(postgres_connection_1.PostgresConnection.prototype, 'getSQLBlockSchema')
+            .spyOn(postgres_connection_1.PostgresConnection.prototype, 'fetchSelectSchema')
             .mockResolvedValue({
-            type: 'struct',
+            type: 'sql select',
             dialect: 'postgres',
             name: 'name',
-            structSource: {
-                type: 'sql',
-                method: 'subquery',
-                sqlBlock: SQL_BLOCK_1,
-            },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: 'postgres',
-            },
+            selectStr: SQL_BLOCK_1.selectStr,
+            connection: 'postgres',
             fields: [],
         });
     });
@@ -92,23 +81,26 @@ describe('PostgresConnection', () => {
         expect(getTableSchema).toBeCalledTimes(2);
     });
     it('caches sql schema', async () => {
-        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+        await connection.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
         expect(getSQLBlockSchema).toBeCalledTimes(1);
-        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_1, {});
+        await connection.fetchSchemaForSQLStruct(SQL_BLOCK_1, {});
         expect(getSQLBlockSchema).toBeCalledTimes(1);
     });
     it('refreshes sql schema', async () => {
-        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {});
+        await connection.fetchSchemaForSQLStruct(SQL_BLOCK_2, {});
         expect(getSQLBlockSchema).toBeCalledTimes(1);
-        await connection.fetchSchemaForSQLBlock(SQL_BLOCK_2, {
+        await connection.fetchSchemaForSQLStruct(SQL_BLOCK_2, {
             refreshTimestamp: Date.now() + 10,
         });
         expect(getSQLBlockSchema).toBeCalledTimes(2);
     });
 });
 const SQL_BLOCK_1 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block1',
+    dialect: 'postgres',
+    connection: 'postgres',
+    fields: [],
     selectStr: `
 SELECT
 created_at,
@@ -124,8 +116,11 @@ FROM "inventory_items.parquet"
 `,
 };
 const SQL_BLOCK_2 = {
-    type: 'sqlBlock',
+    type: 'sql_select',
     name: 'block2',
+    dialect: 'postgres',
+    connection: 'postgres',
+    fields: [],
     selectStr: `
 SELECT
 created_at,

package/dist/postgres_connection.d.ts

@@ -1,4 +1,4 @@
-import { Connection, ConnectionConfig, FetchSchemaOptions, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, SQLBlock, StreamingConnection, StructDef } from '@malloydata/malloy';
+import { Connection, ConnectionConfig, MalloyQueryData, PersistSQLResults, PooledConnection, QueryDataRow, QueryOptionsReader, QueryRunStats, RunSQLOptions, SQLSourceDef, TableSourceDef, StreamingConnection } from '@malloydata/malloy';
 import { BaseConnection } from '@malloydata/malloy/connection';
 import { Client, Pool } from 'pg';
 interface PostgresConnectionConfiguration {
@@ -17,8 +17,6 @@ export declare class PostgresConnection extends BaseConnection implements Connec
     private queryOptionsReader;
     private configReader;
     private readonly dialect;
-    private schemaCache;
-    private sqlSchemaCache;
     constructor(options: PostgresConnectionOptions, queryOptionsReader?: QueryOptionsReader);
     constructor(name: string, queryOptionsReader?: QueryOptionsReader, configReader?: PostgresConnectionConfigurationReader);
     private readQueryConfig;
@@ -28,22 +26,11 @@ export declare class PostgresConnection extends BaseConnection implements Connec
     canPersist(): this is PersistSQLResults;
     canStream(): this is StreamingConnection;
     get supportsNesting(): boolean;
-    fetchSchemaForTables(missing: Record<string, string>, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        schemas: Record<string, StructDef>;
-        errors: Record<string, string>;
-    }>;
-    fetchSchemaForSQLBlock(sqlRef: SQLBlock, { refreshTimestamp }: FetchSchemaOptions): Promise<{
-        structDef: StructDef;
-        error?: undefined;
-    } | {
-        error: string;
-        structDef?: undefined;
-    }>;
     protected getClient(): Promise<Client>;
     protected runPostgresQuery(sqlCommand: string, _pageSize: number, _rowIndex: number, deJSON: boolean): Promise<MalloyQueryData>;
-    private getSQLBlockSchema;
+    fetchSelectSchema(sqlRef: SQLSourceDef): Promise<SQLSourceDef | string>;
     private schemaFromQuery;
-    private getTableSchema;
+    fetchTableSchema(tableKey: string, tablePath: string): Promise<TableSourceDef | string>;
     test(): Promise<void>;
     connectionSetup(client: Client): Promise<void>;
     runSQL(sql: string, { rowLimit }?: RunSQLOptions, rowIndex?: number): Promise<MalloyQueryData>;

package/dist/postgres_connection.js

@@ -68,8 +68,6 @@ class PostgresConnection extends connection_1.BaseConnection {
         this.queryOptionsReader = {};
         this.configReader = {};
         this.dialect = new malloy_1.PostgresDialect();
-        this.schemaCache = new Map();
-        this.sqlSchemaCache = new Map();
         if (typeof arg === 'string') {
             this.name = arg;
             if (configReader) {
@@ -116,54 +114,6 @@ class PostgresConnection extends connection_1.BaseConnection {
     get supportsNesting() {
         return true;
     }
-    async fetchSchemaForTables(missing, { refreshTimestamp }) {
-        const schemas = {};
-        const errors = {};
-        for (const tableKey in missing) {
-            let inCache = this.schemaCache.get(tableKey);
-            if (!inCache ||
-                (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-                const tablePath = missing[tableKey];
-                const timestamp = refreshTimestamp || Date.now();
-                try {
-                    inCache = {
-                        schema: await this.getTableSchema(tableKey, tablePath),
-                        timestamp,
-                    };
-                    this.schemaCache.set(tableKey, inCache);
-                }
-                catch (error) {
-                    inCache = { error: error.message, timestamp };
-                }
-            }
-            if (inCache.schema !== undefined) {
-                schemas[tableKey] = inCache.schema;
-            }
-            else {
-                errors[tableKey] = inCache.error || 'Unknown schema fetch error';
-            }
-        }
-        return { schemas, errors };
-    }
-    async fetchSchemaForSQLBlock(sqlRef, { refreshTimestamp }) {
-        const key = sqlRef.name;
-        let inCache = this.sqlSchemaCache.get(key);
-        if (!inCache ||
-            (refreshTimestamp && refreshTimestamp > inCache.timestamp)) {
-            const timestamp = refreshTimestamp !== null && refreshTimestamp !== void 0 ? refreshTimestamp : Date.now();
-            try {
-                inCache = {
-                    structDef: await this.getSQLBlockSchema(sqlRef),
-                    timestamp,
-                };
-            }
-            catch (error) {
-                inCache = { error: error.message, timestamp };
-            }
-            this.sqlSchemaCache.set(key, inCache);
-        }
-        return inCache;
-    }
     async getClient() {
         const { username: user, password, databaseName: database, port, host, connectionString, } = await this.readConfig();
         return new pg_1.Client({
@@ -194,22 +144,8 @@ class PostgresConnection extends connection_1.BaseConnection {
             totalRows: result.rows.length,
         };
     }
-    async getSQLBlockSchema(sqlRef) {
-        const structDef = {
-            type: 'struct',
-            dialect: 'postgres',
-            name: sqlRef.name,
-            structSource: {
-                type: 'sql',
-                method: 'subquery',
-                sqlBlock: sqlRef,
-            },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
-            fields: [],
-        };
+    async fetchSelectSchema(sqlRef) {
+        const structDef = { ...sqlRef, fields: [] };
         const tempTableName = `tmp${(0, crypto_1.randomUUID)()}`.replace(/-/g, '');
         const infoQuery = `
       drop table if exists ${tempTableName};
@@ -226,7 +162,7 @@ class PostgresConnection extends connection_1.BaseConnection {
             await this.schemaFromQuery(infoQuery, structDef);
         }
         catch (error) {
-            throw new Error(`Error fetching schema for ${sqlRef.name}: ${error}`);
+            return `Error fetching schema for ${sqlRef.name}: ${error}`;
         }
         return structDef;
     }
@@ -237,53 +173,36 @@ class PostgresConnection extends connection_1.BaseConnection {
         }
         for (const row of rows) {
             const postgresDataType = row['data_type'];
-            let s = structDef;
-            let malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType);
-            let name = row['column_name'];
+            const name = row['column_name'];
             if (postgresDataType === 'ARRAY') {
-                malloyType = this.dialect.sqlTypeToMalloyType(row['element_type']);
-                s = {
-                    type: 'struct',
-                    name: row['column_name'],
+                const elementType = this.dialect.sqlTypeToMalloyType(row['element_type']);
+                structDef.fields.push({
+                    type: 'array',
+                    elementTypeDef: elementType,
+                    name,
                     dialect: this.dialectName,
-                    structRelationship: {
-                        type: 'nested',
-                        fieldName: name,
-                        isArray: true,
-                    },
-                    structSource: { type: 'nested' },
-                    fields: [],
-                };
-                structDef.fields.push(s);
-                name = 'value';
-            }
-            if (malloyType) {
-                s.fields.push({ ...malloyType, name });
+                    join: 'many',
+                    fields: (0, malloy_1.arrayEachFields)(elementType),
+                });
             }
             else {
-                s.fields.push({
-                    type: 'sql native',
-                    rawType: postgresDataType.toLowerCase(),
-                    name,
-                });
+                const malloyType = this.dialect.sqlTypeToMalloyType(postgresDataType);
+                structDef.fields.push({ ...malloyType, name });
             }
         }
     }
-    async getTableSchema(tableKey, tablePath) {
+    async fetchTableSchema(tableKey, tablePath) {
         const structDef = {
-            type: 'struct',
+            type: 'table',
             name: tableKey,
             dialect: 'postgres',
-            structSource: { type: 'table', tablePath },
-            structRelationship: {
-                type: 'basetable',
-                connectionName: this.name,
-            },
+            tablePath,
+            connection: this.name,
             fields: [],
         };
         const [schema, table] = tablePath.split('.');
         if (table === undefined) {
-            throw new Error('Default schema not yet supported in Postgres');
+            return 'Default schema not yet supported in Postgres';
         }
         const infoQuery = `
       SELECT column_name, c.data_type, e.data_type as element_type
@@ -297,7 +216,7 @@ class PostgresConnection extends connection_1.BaseConnection {
             await this.schemaFromQuery(infoQuery, structDef);
         }
         catch (error) {
-            throw new Error(`Error fetching schema for ${tablePath}: ${error}`);
+            return `Error fetching schema for ${tablePath}: ${error.message}`;
         }
         return structDef;
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@malloydata/db-postgres",
-  "version": "0.0.195-dev241003204819",
+  "version": "0.0.195-dev241007154000",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -22,7 +22,7 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
-    "@malloydata/malloy": "^0.0.195-dev241003204819",
+    "@malloydata/malloy": "^0.0.195-dev241007154000",
     "@types/pg": "^8.6.1",
     "pg": "^8.7.1",
     "pg-query-stream": "4.2.3"