@malloy-publisher/server 0.0.119 → 0.0.121

package/src/service/db_utils.ts

@@ -1,141 +1,90 @@
 import { BigQuery } from "@google-cloud/bigquery";
-import fs from "fs";
-import os from "os";
-import path from "path";
-import { Pool } from "pg";
-import * as snowflake from "snowflake-sdk";
-import { v4 as uuidv4 } from "uuid";
+import { Connection, TableSourceDef } from "@malloydata/malloy";
 import { components } from "../api";
+import { ConnectionError } from "../errors";
 import { logger } from "../logger";
-import {
-   ApiConnection,
-   // @ts-expect-error TODO: Fix missing MysqlConnection type in API
-   MysqlConnection,
-   PostgresConnection,
-   SnowflakeConnection,
-   TrinoConnection,
-} from "./model";
+import { ApiConnection } from "./model";
 
-import { BasicAuth, Trino } from "trino-client";
+type ApiSchema = components["schemas"]["Schema"];
+type ApiTable = components["schemas"]["Table"];
+type ApiTableSource = components["schemas"]["TableSource"];
 
-type ApiSchemaName = components["schemas"]["SchemaName"];
-
-async function getPostgresConnection(
-   apiPostgresConnection: PostgresConnection,
-): Promise<Pool> {
-   return new Pool({
-      user: apiPostgresConnection.userName,
-      host: apiPostgresConnection.host,
-      database: apiPostgresConnection.databaseName,
-      password: apiPostgresConnection.password,
-      port: apiPostgresConnection.port,
-      connectionString: apiPostgresConnection.connectionString,
-      max: 10,
-      idleTimeoutMillis: 30000,
-   });
-}
+function createBigQueryClient(connection: ApiConnection): BigQuery {
+   if (!connection.bigqueryConnection) {
+      throw new Error("BigQuery connection is required");
+   }
 
-async function getMysqlConnection(apiMysqlConnection: MysqlConnection) {
-   // Dynamically import mysql2/promise to avoid import issues if not needed
-   const mysql = await import("mysql2/promise");
-   return mysql.createPool({
-      host: apiMysqlConnection.host,
-      port: apiMysqlConnection.port,
-      user: apiMysqlConnection.user,
-      password: apiMysqlConnection.password,
-      database: apiMysqlConnection.database,
-      waitForConnections: true,
-      connectionLimit: 10,
-      queueLimit: 0,
-   });
-}
+   const config: {
+      projectId: string;
+      credentials?: object;
+      keyFilename?: string;
+   } = {
+      projectId: connection.bigqueryConnection.defaultProjectId || "",
+   };
 
-function getBigqueryConnection(apiConnection: ApiConnection): BigQuery {
-   if (!apiConnection.bigqueryConnection?.serviceAccountKeyJson) {
-      // Use default credentials
-      return new BigQuery();
-   } else {
-      const tmpKeyPath = getTempServiceKeyPath(apiConnection);
-      if (!tmpKeyPath) {
-         throw new Error(
-            `Failed to create temporary service key file for connection: ${apiConnection.name}`,
+   // Add service account key if provided
+   if (connection.bigqueryConnection.serviceAccountKeyJson) {
+      try {
+         const credentials = JSON.parse(
+            connection.bigqueryConnection.serviceAccountKeyJson,
          );
-      }
-      return new BigQuery({ keyFilename: tmpKeyPath });
-   }
-}
+         config.credentials = credentials;
 
-async function getSnowflakeConnection(
-   apiSnowflakeConnection: SnowflakeConnection,
-): Promise<snowflake.Connection> {
-   if (!apiSnowflakeConnection.account) {
-      throw new Error("Snowflake account is required");
-   }
-   return new Promise((resolve, reject) => {
-      const connection = snowflake.createConnection({
-         account: apiSnowflakeConnection.account as string,
-         username: apiSnowflakeConnection.username,
-         password: apiSnowflakeConnection.password,
-         database: apiSnowflakeConnection.database,
-         warehouse: apiSnowflakeConnection.warehouse || undefined,
-         role: apiSnowflakeConnection.role || undefined,
-      });
-      connection.connect((err, conn) => {
-         if (err) {
-            reject(err);
-         } else {
-            resolve(conn);
+         // Use project_id from credentials if defaultProjectId is not set
+         if (!config.projectId && credentials.project_id) {
+            config.projectId = credentials.project_id;
          }
-      });
-   });
-}
 
-function getTrinoClient(trinoConn: TrinoConnection) {
-   let auth: BasicAuth;
-   if (trinoConn.server?.startsWith("https://")) {
-      // HTTPS allows password authentication
-      auth = new BasicAuth(trinoConn?.user || "", trinoConn?.password || "");
+         if (!config.projectId) {
+            throw new Error(
+               "BigQuery project ID is required. Either set the defaultProjectId in the connection configuration or the project_id in the service account key JSON.",
+            );
+         }
+      } catch (error) {
+         logger.warn(
+            "Failed to parse service account key JSON, using default credentials",
+            { error },
+         );
+      }
+   } else if (
+      Object.keys(connection.bigqueryConnection).length === 0 &&
+      process.env.GOOGLE_APPLICATION_CREDENTIALS
+   ) {
+      // Note: The BigQuery client will infer the project ID from the ADC file.
+      config.keyFilename = process.env.GOOGLE_APPLICATION_CREDENTIALS || "";
    } else {
-      // HTTP only allows username, no password for security
-      auth = new BasicAuth(trinoConn?.user || "");
+      throw new Error(
+         "BigQuery connection is required, either set the bigqueryConnection in the connection configuration or set the GOOGLE_APPLICATION_CREDENTIALS environment variable.",
+      );
    }
 
-   return Trino.create({
-      server: trinoConn.server,
-      catalog: trinoConn.catalog,
-      schema: trinoConn.schema,
-      auth,
-   });
+   return new BigQuery(config);
+}
+
+function standardizeRunSQLResult(result: unknown): unknown[] {
+   // Handle different result formats from malloyConnection.runSQL
+   return Array.isArray(result)
+      ? result
+      : (result as { rows?: unknown[] }).rows || [];
 }
 
 export async function getSchemasForConnection(
    connection: ApiConnection,
-): Promise<ApiSchemaName[]> {
+   malloyConnection: Connection,
+): Promise<ApiSchema[]> {
    if (connection.type === "bigquery") {
       if (!connection.bigqueryConnection) {
          throw new Error("BigQuery connection is required");
       }
       try {
-         const bigquery = getBigqueryConnection(connection);
-         const projectId = connection.bigqueryConnection.defaultProjectId;
-         const options = projectId ? { projectId } : {};
-         const [datasets] = await bigquery.getDatasets(options);
-         const schemas = await Promise.all(
-            datasets
-               .filter((dataset) => dataset.id)
-               .map(async (dataset) => {
-                  const [metadata] = await dataset.getMetadata();
-                  return {
-                     name: dataset.id,
-                     isHidden: false,
-                     isDefault: false,
-                     // Include description from dataset metadata if available
-                     description: (metadata as { description?: string })
-                        ?.description,
-                  };
-               }),
-         );
-         return schemas;
+         const bigquery = createBigQueryClient(connection);
+         const [datasets] = await bigquery.getDatasets();
+
+         return datasets.map((dataset) => ({
+            name: dataset.id,
+            isHidden: false,
+            isDefault: false,
+         }));
       } catch (error) {
          console.error(
             `Error getting schemas for BigQuery connection ${connection.name}:`,
@@ -149,67 +98,161 @@ export async function getSchemasForConnection(
       if (!connection.postgresConnection) {
          throw new Error("Postgres connection is required");
       }
-      const pool = await getPostgresConnection(connection.postgresConnection);
-      const res = await pool.query(
-         "SELECT schema_name FROM information_schema.schemata",
-      );
-      return res.rows.map((row) => {
-         return {
-            name: row.schema_name,
-            isHidden: ["information_schema", "pg_catalog"].includes(
-               row.schema_name,
-            ),
-            isDefault: row.schema_name === "public",
-         };
-      });
+      try {
+         // Use the connection's runSQL method to query schemas
+         const result = await malloyConnection.runSQL(
+            "SELECT schema_name as row FROM information_schema.schemata ORDER BY schema_name",
+         );
+
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const schemaName = row as string;
+            return {
+               name: schemaName,
+               isHidden: [
+                  "information_schema",
+                  "pg_catalog",
+                  "pg_toast",
+               ].includes(schemaName),
+               isDefault: schemaName === "public",
+            };
+         });
+      } catch (error) {
+         console.error(
+            `Error getting schemas for Postgres connection ${connection.name}:`,
+            error,
+         );
+         throw new Error(
+            `Failed to get schemas for Postgres connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
    } else if (connection.type === "mysql") {
       if (!connection.mysqlConnection) {
          throw new Error("Mysql connection is required");
       }
-      return [
-         {
-            name: connection.mysqlConnection.database || "mysql",
-            isHidden: false,
-            isDefault: true,
-         },
-      ];
+      try {
+         // For MySQL, return the database name as the schema
+         return [
+            {
+               name: connection.mysqlConnection.database || "mysql",
+               isHidden: false,
+               isDefault: true,
+            },
+         ];
+      } catch (error) {
+         console.error(
+            `Error getting schemas for MySQL connection ${connection.name}:`,
+            error,
+         );
+         throw new Error(
+            `Failed to get schemas for MySQL connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
    } else if (connection.type === "snowflake") {
       if (!connection.snowflakeConnection) {
          throw new Error("Snowflake connection is required");
       }
-      const snowflakeConn = await getSnowflakeConnection(
-         connection.snowflakeConnection,
-      );
       try {
-         return await getSnowflakeSchemas(snowflakeConn);
-      } finally {
-         snowflakeConn.destroy((error) => {
-            if (error) {
-               logger.error(`Error closing SnowflakeConnection: ${error}`);
-            }
+         // Use the connection's runSQL method to query schemas
+         const result = await malloyConnection.runSQL("SHOW SCHEMAS");
+
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return {
+               name: typedRow.name as string,
+               isHidden: ["SNOWFLAKE", ""].includes(typedRow.owner as string),
+               isDefault: typedRow.isDefault === "Y",
+            };
          });
+      } catch (error) {
+         console.error(
+            `Error getting schemas for Snowflake connection ${connection.name}:`,
+            error,
+         );
+         throw new Error(
+            `Failed to get schemas for Snowflake connection ${connection.name}: ${(error as Error).message}`,
+         );
       }
    } else if (connection.type === "trino") {
       if (!connection.trinoConnection) {
          throw new Error("Trino connection is required");
       }
-      const client = getTrinoClient(connection.trinoConnection);
-      const result = await client.query(
-         `SHOW SCHEMAS FROM ${connection.trinoConnection.catalog}`,
-      );
-      const rows: string[] = [];
-      let next = await result.next();
-      while (!next.done) {
-         if (next.value.data) {
-            rows.push(...next.value.data.map((r: string[]) => r[0]));
-         }
-         next = await result.next();
+      try {
+         // Use the connection's runSQL method to query schemas
+         const result = await malloyConnection.runSQL(
+            `SHOW SCHEMAS FROM ${connection.trinoConnection.catalog}`,
+         );
+
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return {
+               name: typedRow.Schema as string,
+               isHidden: ["information_schema", "performance_schema"].includes(
+                  typedRow.Schema as string,
+               ),
+               isDefault:
+                  typedRow.Schema === connection.trinoConnection?.schema,
+            };
+         });
+      } catch (error) {
+         console.error(
+            `Error getting schemas for Trino connection ${connection.name}:`,
+            error,
+         );
+         throw new Error(
+            `Failed to get schemas for Trino connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
+   } else if (connection.type === "duckdb") {
+      if (!connection.duckdbConnection) {
+         throw new Error("DuckDB connection is required");
+      }
+      try {
+         // Use DuckDB's INFORMATION_SCHEMA.SCHEMATA to list schemas
+         // Use DISTINCT to avoid duplicates from attached databases
+         const result = await malloyConnection.runSQL(
+            "SELECT DISTINCT schema_name FROM information_schema.schemata ORDER BY schema_name",
+         );
+
+         const rows = standardizeRunSQLResult(result);
+
+         // Check if this DuckDB connection has attached databases
+         const hasAttachedDatabases =
+            connection.duckdbConnection?.attachedDatabases &&
+            Array.isArray(connection.duckdbConnection.attachedDatabases) &&
+            connection.duckdbConnection.attachedDatabases.length > 0;
+
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            let schemaName = typedRow.schema_name as string;
+
+            // If we have attached databases and this is not the main schema, prepend the attached database name
+            if (hasAttachedDatabases && schemaName !== "main") {
+               const attachedDbName = (
+                  connection.duckdbConnection!.attachedDatabases as Array<{
+                     name: string;
+                  }>
+               )[0].name;
+               schemaName = `${attachedDbName}.${schemaName}`;
+            }
+
+            return {
+               name: schemaName,
+               isHidden: false,
+               isDefault: typedRow.schema_name === "main",
+            };
+         });
+      } catch (error) {
+         console.error(
+            `Error getting schemas for DuckDB connection ${connection.name}:`,
+            error,
+         );
+         throw new Error(
+            `Failed to get schemas for DuckDB connection ${connection.name}: ${(error as Error).message}`,
+         );
       }
-      return rows.map((r) => ({
-         name: r,
-         isHidden: false,
-         isDefault: r === connection.trinoConnection?.schema,
-      }));
    } else {
       throw new Error(`Unsupported connection type: ${connection.type}`);
    }
@@ -218,25 +261,113 @@ export async function getSchemasForConnection(
 export async function getTablesForSchema(
    connection: ApiConnection,
    schemaName: string,
-): Promise<string[]> {
-   if (connection.type === "bigquery") {
+   malloyConnection: Connection,
+): Promise<ApiTable[]> {
+   // First get the list of table names
+   const tableNames = await listTablesForSchema(
+      connection,
+      schemaName,
+      malloyConnection,
+   );
+
+   // Fetch all table sources in parallel
+   const tableSourcePromises = tableNames.map(async (tableName) => {
       try {
-         const bigquery = getBigqueryConnection(connection);
-         const options = connection.bigqueryConnection?.defaultProjectId
-            ? {
-                 projectId: connection.bigqueryConnection?.defaultProjectId,
-              }
-            : {};
-         const dataset = bigquery.dataset(schemaName, options);
-         const [exists] = await dataset.exists();
-         if (!exists) {
-            throw new Error(
-               `Dataset ${schemaName} does not exist in connection ${connection.name}`,
-            );
+         const tablePath = `${schemaName}.${tableName}`;
+
+         logger.info(
+            `Processing table: ${tableName} in schema: ${schemaName}`,
+            { tablePath, connectionType: connection.type },
+         );
+         const tableSource = await getConnectionTableSource(
+            malloyConnection,
+            tableName,
+            tablePath,
+         );
+
+         return {
+            resource: tablePath,
+            columns: tableSource.columns,
+         };
+      } catch (error) {
+         logger.warn(`Failed to get schema for table ${tableName}`, {
+            error,
+            schemaName,
+            tableName,
+         });
+         // Return table without columns if schema fetch fails
+         return {
+            resource: `${schemaName}.${tableName}`,
+            columns: [],
+         };
+      }
+   });
+
+   // Wait for all table sources to be fetched
+   const tableResults = await Promise.all(tableSourcePromises);
+
+   return tableResults;
+}
+
+export async function getConnectionTableSource(
+   malloyConnection: Connection,
+   tableKey: string,
+   tablePath: string,
+): Promise<ApiTableSource> {
+   try {
+      logger.info(`Attempting to fetch table schema for: ${tablePath}`, {
+         tableKey,
+         tablePath,
+      });
+      const source = await (
+         malloyConnection as Connection & {
+            fetchTableSchema: (
+               tableKey: string,
+               tablePath: string,
+            ) => Promise<TableSourceDef | undefined>;
          }
+      ).fetchTableSchema(tableKey, tablePath);
+      if (source === undefined) {
+         throw new ConnectionError(`Table ${tablePath} not found`);
+      }
+      const malloyFields = (source as TableSourceDef).fields;
+      const fields = malloyFields.map((field) => {
+         return {
+            name: field.name,
+            type: field.type,
+         };
+      });
+      logger.info(`Successfully fetched schema for ${tablePath}`, {
+         fieldCount: fields.length,
+      });
+      return {
+         source: JSON.stringify(source),
+         resource: tablePath,
+         columns: fields,
+      };
+   } catch (error) {
+      logger.error("fetchTableSchema error", { error, tableKey, tablePath });
+      throw new ConnectionError((error as Error).message);
+   }
+}
 
+export async function listTablesForSchema(
+   connection: ApiConnection,
+   schemaName: string,
+   malloyConnection: Connection,
+): Promise<string[]> {
+   if (connection.type === "bigquery") {
+      try {
+         // Use BigQuery client directly for efficient table listing
+         // This is much faster than querying all regions
+         const bigquery = createBigQueryClient(connection);
+         const dataset = bigquery.dataset(schemaName);
          const [tables] = await dataset.getTables();
-         return tables.map((table) => table.id).filter((id) => id) as string[];
+
+         // Return table names, filtering out any undefined values
+         return tables
+            .map((table) => table.id)
+            .filter((id): id is string => id !== undefined);
       } catch (error) {
          logger.error(
             `Error getting tables for BigQuery schema ${schemaName} in connection ${connection.name}`,
@@ -250,153 +381,137 @@ export async function getTablesForSchema(
       if (!connection.mysqlConnection) {
          throw new Error("Mysql connection is required");
       }
-      const pool = await getMysqlConnection(connection.mysqlConnection);
-      const [rows] = await pool.query(
-         "SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = ? AND table_type = 'BASE TABLE'",
-         [schemaName],
-      );
-      return (rows as { TABLE_NAME: string }[]).map((row) => row.TABLE_NAME);
+      try {
+         const result = await malloyConnection.runSQL(
+            `SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = '${schemaName}' AND table_type = 'BASE TABLE'`,
+         );
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return typedRow.TABLE_NAME as string;
+         });
+      } catch (error) {
+         logger.error(
+            `Error getting tables for MySQL schema ${schemaName} in connection ${connection.name}`,
+            { error },
+         );
+         throw new Error(
+            `Failed to get tables for MySQL schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
    } else if (connection.type === "postgres") {
       if (!connection.postgresConnection) {
          throw new Error("Postgres connection is required");
       }
-      const pool = await getPostgresConnection(connection.postgresConnection);
-      const res = await pool.query(
-         "SELECT table_name FROM information_schema.tables WHERE table_schema = $1",
-         [schemaName],
-      );
-      return res.rows.map((row) => row.table_name);
+      try {
+         const result = await malloyConnection.runSQL(
+            `SELECT table_name as row FROM information_schema.tables WHERE table_schema = '${schemaName}' ORDER BY table_name`,
+         );
+         const rows = standardizeRunSQLResult(result);
+         return rows as string[];
+      } catch (error) {
+         logger.error(
+            `Error getting tables for Postgres schema ${schemaName} in connection ${connection.name}`,
+            { error },
+         );
+         throw new Error(
+            `Failed to get tables for Postgres schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
    } else if (connection.type === "snowflake") {
       if (!connection.snowflakeConnection) {
          throw new Error("Snowflake connection is required");
       }
-      const snowflakeConn = await getSnowflakeConnection(
-         connection.snowflakeConnection,
-      );
       try {
-         return await getSnowflakeTables(
-            snowflakeConn,
-            connection.snowflakeConnection,
-            schemaName,
+         const result = await malloyConnection.runSQL(
+            `SELECT TABLE_NAME FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = '${schemaName}' AND TABLE_TYPE = 'BASE TABLE'`,
          );
-      } finally {
-         snowflakeConn.destroy((error) => {
-            if (error) {
-               logger.error(`Error closing SnowflakeConnection`, { error });
-            }
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return typedRow.TABLE_NAME as string;
          });
+      } catch (error) {
+         logger.error(
+            `Error getting tables for Snowflake schema ${schemaName} in connection ${connection.name}`,
+            { error },
+         );
+         throw new Error(
+            `Failed to get tables for Snowflake schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+         );
       }
    } else if (connection.type === "trino") {
       if (!connection.trinoConnection) {
          throw new Error("Trino connection is required");
       }
-      const client = getTrinoClient(connection.trinoConnection);
-      const result = await client.query(
-         `SHOW TABLES FROM ${connection.trinoConnection.catalog}.${schemaName}`,
-      );
+      try {
+         const result = await malloyConnection.runSQL(
+            `SHOW TABLES FROM ${connection.trinoConnection.catalog}.${schemaName}`,
+         );
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return typedRow.Table as string;
+         });
+      } catch (error) {
+         logger.error(
+            `Error getting tables for Trino schema ${schemaName} in connection ${connection.name}`,
+            { error },
+         );
+         throw new Error(
+            `Failed to get tables for Trino schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+         );
+      }
+   } else if (connection.type === "duckdb") {
+      if (!connection.duckdbConnection) {
+         throw new Error("DuckDB connection is required");
+      }
+      try {
+         // Check if this DuckDB connection has attached databases and if the schema name is prepended
+         const hasAttachedDatabases =
+            connection.duckdbConnection?.attachedDatabases &&
+            Array.isArray(connection.duckdbConnection.attachedDatabases) &&
+            connection.duckdbConnection.attachedDatabases.length > 0;
+
+         let actualSchemaName = schemaName;
 
-      const rows: string[] = [];
-      let next = await result.next();
-      while (!next.done) {
-         if (next.value.data) {
-            rows.push(...next.value.data.map((r: string[]) => r[0]));
+         // If we have attached databases and the schema name is prepended, extract the actual schema name
+         if (hasAttachedDatabases && schemaName.includes(".")) {
+            const attachedDbName = (
+               connection.duckdbConnection!.attachedDatabases as Array<{
+                  name: string;
+               }>
+            )[0].name;
+            if (schemaName.startsWith(`${attachedDbName}.`)) {
+               actualSchemaName = schemaName.substring(
+                  attachedDbName.length + 1,
+               );
+            }
          }
-         next = await result.next();
+
+         // Use DuckDB's INFORMATION_SCHEMA.TABLES to list tables in the specified schema
+         // This follows the DuckDB documentation for listing tables
+         // For DuckDB, we'll use string interpolation to avoid parameter binding issues
+         const result = await malloyConnection.runSQL(
+            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${actualSchemaName}' ORDER BY table_name`,
+         );
+
+         const rows = standardizeRunSQLResult(result);
+         return rows.map((row: unknown) => {
+            const typedRow = row as Record<string, unknown>;
+            return typedRow.table_name as string;
+         });
+      } catch (error) {
+         logger.error(
+            `Error getting tables for DuckDB schema ${schemaName} in connection ${connection.name}`,
+            { error },
+         );
+         throw new Error(
+            `Failed to get tables for DuckDB schema ${schemaName} in connection ${connection.name}: ${(error as Error).message}`,
+         );
       }
-      return rows;
    } else {
       // TODO(jjs) - implement
       return [];
    }
 }
-
-function getTempServiceKeyPath(connection: ApiConnection): string {
-   // If the connection is bigquery and the service account key is provided as
-   // JSON, we need to write it to a temporary file for the Malloy compiling to work
-   if (!connection.bigqueryConnection) {
-      throw new Error(
-         `BigQuery connection ${connection.name} is missing bigqueryConnection ${JSON.stringify(connection)}`,
-      );
-   }
-   const keyJson = connection.bigqueryConnection.serviceAccountKeyJson;
-   if (!keyJson) {
-      throw new Error(
-         `BigQuery connection ${connection.name} is missing service account key JSON`,
-      );
-   }
-   const tmpFilepath = path.join(
-      os.tmpdir(),
-      `ms2_connection_${connection.name}_${uuidv4()}_key.json`,
-   );
-   fs.writeFileSync(tmpFilepath, keyJson, "utf8");
-   return tmpFilepath;
-}
-
-async function getSnowflakeTables(
-   connection: snowflake.Connection,
-   connInfo: SnowflakeConnection,
-   schemaName: string,
-): Promise<string[]> {
-   return new Promise((resolve, reject) => {
-      connection.execute({
-         sqlText: `USE DATABASE ${connInfo?.database} `,
-         complete: (err) => {
-            if (err) {
-               logger.error(`Error setting database ${connInfo.database}:`, {
-                  error: err,
-               });
-               reject([]);
-               return;
-            }
-
-            const query = `
-          SELECT TABLE_NAME, TABLE_SCHEMA
-          FROM INFORMATION_SCHEMA.TABLES
-          WHERE TABLE_SCHEMA=? AND TABLE_TYPE = 'BASE TABLE';
-        `;
-
-            connection.execute({
-               sqlText: query,
-               binds: [schemaName],
-               complete: (err, _, rows) => {
-                  if (err) {
-                     logger.error(
-                        `Error fetching tables from ${connInfo.database}:`,
-                        { error: err },
-                     );
-                     reject([]);
-                  } else {
-                     resolve(rows?.map((row) => `${row.TABLE_NAME}`) || []);
-                  }
-               },
-            });
-         },
-      });
-   });
-}
-
-async function getSnowflakeSchemas(
-   connection: snowflake.Connection,
-): Promise<ApiSchemaName[]> {
-   return new Promise((resolve, reject) => {
-      connection.execute({
-         sqlText: "SHOW SCHEMAS",
-         complete: (err, _stmt, rows) => {
-            if (err) {
-               reject(err);
-            } else {
-               resolve(
-                  rows?.map((row) => {
-                     logger.info("row", { row });
-                     return {
-                        name: row.name,
-                        isDefault: row.isDefault === "Y",
-                        isHidden: ["SNOWFLAKE", ""].includes(row.owner),
-                     };
-                  }) || [],
-               );
-            }
-         },
-      });
-   });
-}