@malloy-publisher/server 0.0.120 → 0.0.121

package/src/service/db_utils.ts

@@ -14,29 +14,48 @@ function createBigQueryClient(connection: ApiConnection): BigQuery {
       throw new Error("BigQuery connection is required");
    }
 
-   if (!connection.bigqueryConnection.defaultProjectId) {
-      throw new Error("BigQuery defaultProjectId is required");
-   }
-
    const config: {
       projectId: string;
+      credentials?: object;
       keyFilename?: string;
    } = {
-      projectId: connection.bigqueryConnection.defaultProjectId,
+      projectId: connection.bigqueryConnection.defaultProjectId || "",
    };
 
    // Add service account key if provided
    if (connection.bigqueryConnection.serviceAccountKeyJson) {
       try {
-         config.keyFilename = JSON.parse(
+         const credentials = JSON.parse(
             connection.bigqueryConnection.serviceAccountKeyJson,
          );
+         config.credentials = credentials;
+
+         // Use project_id from credentials if defaultProjectId is not set
+         if (!config.projectId && credentials.project_id) {
+            config.projectId = credentials.project_id;
+         }
+
+         if (!config.projectId) {
+            throw new Error(
+               "BigQuery project ID is required. Either set the defaultProjectId in the connection configuration or the project_id in the service account key JSON.",
+            );
+         }
       } catch (error) {
          logger.warn(
             "Failed to parse service account key JSON, using default credentials",
             { error },
          );
       }
+   } else if (
+      Object.keys(connection.bigqueryConnection).length === 0 &&
+      process.env.GOOGLE_APPLICATION_CREDENTIALS
+   ) {
+      // Note: The BigQuery client will infer the project ID from the ADC file.
+      config.keyFilename = process.env.GOOGLE_APPLICATION_CREDENTIALS || "";
+   } else {
+      throw new Error(
+         "BigQuery connection is required, either set the bigqueryConnection in the connection configuration or set the GOOGLE_APPLICATION_CREDENTIALS environment variable.",
+      );
    }
 
    return new BigQuery(config);
@@ -58,11 +77,6 @@ export async function getSchemasForConnection(
          throw new Error("BigQuery connection is required");
       }
       try {
-         const projectId = connection.bigqueryConnection?.defaultProjectId;
-         if (!projectId) {
-            throw new Error("BigQuery project ID is required");
-         }
-
          const bigquery = createBigQueryClient(connection);
          const [datasets] = await bigquery.getDatasets();
 
@@ -87,18 +101,20 @@ export async function getSchemasForConnection(
       try {
          // Use the connection's runSQL method to query schemas
          const result = await malloyConnection.runSQL(
-            "SELECT schema_name FROM information_schema.schemata",
+            "SELECT schema_name as row FROM information_schema.schemata ORDER BY schema_name",
          );
 
          const rows = standardizeRunSQLResult(result);
          return rows.map((row: unknown) => {
-            const typedRow = row as Record<string, unknown>;
+            const schemaName = row as string;
             return {
-               name: typedRow.schema_name as string,
-               isHidden: ["information_schema", "pg_catalog"].includes(
-                  typedRow.schema_name as string,
-               ),
-               isDefault: typedRow.schema_name === "public",
+               name: schemaName,
+               isHidden: [
+                  "information_schema",
+                  "pg_catalog",
+                  "pg_toast",
+               ].includes(schemaName),
+               isDefault: schemaName === "public",
             };
          });
       } catch (error) {
@@ -172,9 +188,12 @@ export async function getSchemasForConnection(
          return rows.map((row: unknown) => {
             const typedRow = row as Record<string, unknown>;
             return {
-               name: typedRow.name as string,
-               isHidden: false,
-               isDefault: typedRow.name === connection.trinoConnection?.schema,
+               name: typedRow.Schema as string,
+               isHidden: ["information_schema", "performance_schema"].includes(
+                  typedRow.Schema as string,
+               ),
+               isDefault:
+                  typedRow.Schema === connection.trinoConnection?.schema,
             };
          });
       } catch (error) {
@@ -339,11 +358,6 @@ export async function listTablesForSchema(
 ): Promise<string[]> {
    if (connection.type === "bigquery") {
       try {
-         const projectId = connection.bigqueryConnection?.defaultProjectId;
-         if (!projectId) {
-            throw new Error("BigQuery project ID is required");
-         }
-
          // Use BigQuery client directly for efficient table listing
          // This is much faster than querying all regions
          const bigquery = createBigQueryClient(connection);
@@ -391,13 +405,10 @@ export async function listTablesForSchema(
       }
       try {
          const result = await malloyConnection.runSQL(
-            `SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}'`,
+            `SELECT table_name as row FROM information_schema.tables WHERE table_schema = '${schemaName}' ORDER BY table_name`,
          );
          const rows = standardizeRunSQLResult(result);
-         return rows.map((row: unknown) => {
-            const typedRow = row as Record<string, unknown>;
-            return typedRow.table_name as string;
-         });
+         return rows as string[];
       } catch (error) {
          logger.error(
             `Error getting tables for Postgres schema ${schemaName} in connection ${connection.name}`,
@@ -440,7 +451,7 @@ export async function listTablesForSchema(
          const rows = standardizeRunSQLResult(result);
          return rows.map((row: unknown) => {
             const typedRow = row as Record<string, unknown>;
-            return typedRow.name as string;
+            return typedRow.Table as string;
          });
       } catch (error) {
          logger.error(