@malloy-publisher/server 0.0.122 → 0.0.124

package/dist/server.js

@@ -131921,138 +131921,211 @@ function validateAndBuildTrinoConfig(trinoConfig) {
     throw new Error(`Invalid Trino connection: expected "http://server:port" (no password) or "https://server:port" (with username and password).`);
   }
 }
+async function installAndLoadExtension(connection, extensionName, fromCommunity = false) {
+  try {
+    const installCommand = fromCommunity ? `FORCE INSTALL '${extensionName}' FROM community;` : `INSTALL ${extensionName};`;
+    await connection.runSQL(installCommand);
+    logger2.info(`${extensionName} extension installed`);
+  } catch (error) {
+    logger2.info(`${extensionName} extension already installed or install skipped`, { error });
+  }
+  await connection.runSQL(`LOAD ${extensionName};`);
+  logger2.info(`${extensionName} extension loaded`);
+}
+async function isDatabaseAttached(connection, dbName) {
+  try {
+    const existingDatabases = await connection.runSQL("SHOW DATABASES");
+    const rows = Array.isArray(existingDatabases) ? existingDatabases : existingDatabases.rows || [];
+    logger2.debug(`Existing databases:`, rows);
+    return rows.some((row) => Object.values(row).some((value) => typeof value === "string" && value === dbName));
+  } catch (error) {
+    logger2.warn(`Failed to check existing databases:`, error);
+    return false;
+  }
+}
+function sanitizeSecretName(name) {
+  return `secret_${name.replace(/[^a-zA-Z0-9_]/g, "_")}`;
+}
+function escapeSQL(value) {
+  return value.replace(/'/g, "''");
+}
+function handleAlreadyAttachedError(error, dbName) {
+  if (error instanceof Error && error.message.includes("already exists")) {
+    logger2.info(`Database ${dbName} is already attached, skipping`);
+  } else {
+    throw error;
+  }
+}
+async function attachBigQuery(connection, attachedDb) {
+  if (!attachedDb.bigqueryConnection) {
+    throw new Error(`BigQuery connection configuration missing for: ${attachedDb.name}`);
+  }
+  const config = attachedDb.bigqueryConnection;
+  let projectId = config.defaultProjectId;
+  let serviceAccountJson;
+  if (config.serviceAccountKeyJson) {
+    const keyData = JSON.parse(config.serviceAccountKeyJson);
+    const requiredFields = [
+      "type",
+      "project_id",
+      "private_key",
+      "client_email"
+    ];
+    for (const field of requiredFields) {
+      if (!keyData[field]) {
+        throw new Error(`Invalid service account key: missing "${field}" field`);
+      }
+    }
+    if (keyData.type !== "service_account") {
+      throw new Error('Invalid service account key: incorrect "type" field');
+    }
+    projectId = keyData.project_id || config.defaultProjectId;
+    serviceAccountJson = config.serviceAccountKeyJson;
+    logger2.info(`Using service account: ${keyData.client_email}`);
+  }
+  if (!projectId || !serviceAccountJson) {
+    throw new Error(`BigQuery project_id and service account key required for: ${attachedDb.name}`);
+  }
+  await installAndLoadExtension(connection, "bigquery", true);
+  const secretName = sanitizeSecretName(`bigquery_${attachedDb.name}`);
+  const escapedJson = escapeSQL(serviceAccountJson);
+  const createSecretCommand = `
+      CREATE OR REPLACE SECRET ${secretName} (
+         TYPE BIGQUERY,
+         SCOPE 'bq://${projectId}',
+         SERVICE_ACCOUNT_JSON '${escapedJson}'
+      );
+   `;
+  await connection.runSQL(createSecretCommand);
+  logger2.info(`Created BigQuery secret: ${secretName} for project: ${projectId}`);
+  const attachCommand = `ATTACH 'project=${projectId}' AS ${attachedDb.name} (TYPE bigquery, READ_ONLY);`;
+  await connection.runSQL(attachCommand);
+  logger2.info(`Successfully attached BigQuery database: ${attachedDb.name}`);
+}
+async function attachSnowflake(connection, attachedDb) {
+  if (!attachedDb.snowflakeConnection) {
+    throw new Error(`Snowflake connection configuration missing for: ${attachedDb.name}`);
+  }
+  const config = attachedDb.snowflakeConnection;
+  const requiredFields = {
+    account: config.account,
+    username: config.username,
+    password: config.password
+  };
+  for (const [field, value] of Object.entries(requiredFields)) {
+    if (!value) {
+      throw new Error(`Snowflake ${field} is required for: ${attachedDb.name}`);
+    }
+  }
+  await installAndLoadExtension(connection, "snowflake", true);
+  try {
+    const version = await connection.runSQL("SELECT snowflake_version();");
+    logger2.info(`Snowflake ADBC driver verified with version:`, version.rows);
+  } catch (error) {
+    throw new Error(`Snowflake ADBC driver verification failed: ${error instanceof Error ? error.message : String(error)}`);
+  }
+  const params = {
+    account: escapeSQL(config.account || ""),
+    user: escapeSQL(config.username || ""),
+    password: escapeSQL(config.password || ""),
+    database: config.database ? escapeSQL(config.database) : undefined,
+    warehouse: config.warehouse ? escapeSQL(config.warehouse) : undefined,
+    schema: config.schema ? escapeSQL(config.schema) : undefined,
+    role: config.role ? escapeSQL(config.role) : undefined
+  };
+  const attachParts = [
+    `account=${params.account}`,
+    `user=${params.user}`,
+    `password=${params.password}`
+  ];
+  if (params.database)
+    attachParts.push(`database=${params.database}`);
+  if (params.warehouse)
+    attachParts.push(`warehouse=${params.warehouse}`);
+  const secretString = `CREATE OR REPLACE SECRET ${attachedDb.name}_secret (
+      TYPE snowflake,
+      ACCOUNT '${params.account}',
+      USER '${params.user}',
+      PASSWORD '${params.password}',
+      DATABASE '${params.database}',
+      WAREHOUSE '${params.warehouse}'
+   );`;
+  await connection.runSQL(secretString);
+  const testresult = await connection.runSQL(`SELECT * FROM snowflake_scan('SELECT 1', '${attachedDb.name}_secret');`);
+  logger2.info(`Testing Snowflake connection:`, testresult.rows);
+  const attachCommand = `ATTACH '${attachedDb.name}' AS ${attachedDb.name} (TYPE snowflake, SECRET ${attachedDb.name}_secret, READ_ONLY);`;
+  await connection.runSQL(attachCommand);
+  logger2.info(`Successfully attached Snowflake database: ${attachedDb.name}`);
+}
+async function attachPostgres(connection, attachedDb) {
+  if (!attachedDb.postgresConnection) {
+    throw new Error(`PostgreSQL connection configuration missing for: ${attachedDb.name}`);
+  }
+  await installAndLoadExtension(connection, "postgres");
+  const config = attachedDb.postgresConnection;
+  let attachString;
+  if (config.connectionString) {
+    attachString = config.connectionString;
+  } else {
+    const parts = [];
+    if (config.host)
+      parts.push(`host=${config.host}`);
+    if (config.port)
+      parts.push(`port=${config.port}`);
+    if (config.databaseName)
+      parts.push(`dbname=${config.databaseName}`);
+    if (config.userName)
+      parts.push(`user=${config.userName}`);
+    if (config.password)
+      parts.push(`password=${config.password}`);
+    if (process.env.PGSSLMODE === "no-verify")
+      parts.push(`sslmode=disable`);
+    attachString = parts.join(" ");
+  }
+  const attachCommand = `ATTACH '${attachString}' AS ${attachedDb.name} (TYPE postgres, READ_ONLY);`;
+  await connection.runSQL(attachCommand);
+  logger2.info(`Successfully attached PostgreSQL database: ${attachedDb.name}`);
+}
+async function attachMotherDuck(connection, attachedDb) {
+  if (!attachedDb.motherDuckConnection) {
+    throw new Error(`MotherDuck connection configuration missing for: ${attachedDb.name}`);
+  }
+  const config = attachedDb.motherDuckConnection;
+  if (!config.database) {
+    throw new Error(`MotherDuck database name is required for: ${attachedDb.name}`);
+  }
+  await installAndLoadExtension(connection, "motherduck");
+  if (config.accessToken) {
+    const escapedToken = escapeSQL(config.accessToken);
+    await connection.runSQL(`SET motherduck_token = '${escapedToken}';`);
+  }
+  const connectionString = `md:${config.database}`;
+  logger2.info(`Connecting to MotherDuck database: ${config.database} as ${attachedDb.name}`);
+  const attachCommand = `ATTACH '${connectionString}' AS ${attachedDb.name} (TYPE motherduck, READ_ONLY);`;
+  await connection.runSQL(attachCommand);
+  logger2.info(`Successfully attached MotherDuck database: ${attachedDb.name}`);
+}
 async function attachDatabasesToDuckDB(duckdbConnection, attachedDatabases) {
+  const attachHandlers = {
+    bigquery: attachBigQuery,
+    snowflake: attachSnowflake,
+    postgres: attachPostgres,
+    motherduck: attachMotherDuck
+  };
   for (const attachedDb of attachedDatabases) {
     try {
+      if (await isDatabaseAttached(duckdbConnection, attachedDb.name || "")) {
+        logger2.info(`Database ${attachedDb.name} is already attached, skipping`);
+        continue;
+      }
+      const handler = attachHandlers[attachedDb.type];
+      if (!handler) {
+        throw new Error(`Unsupported database type: ${attachedDb.type}`);
+      }
       try {
-        const checkQuery = `SHOW DATABASES`;
-        const existingDatabases = await duckdbConnection.runSQL(checkQuery);
-        const rows = Array.isArray(existingDatabases) ? existingDatabases : existingDatabases.rows || [];
-        logger2.debug(`Existing databases:`, rows);
-        const isAlreadyAttached = rows.some((row) => {
-          return Object.values(row).some((value) => typeof value === "string" && value === attachedDb.name);
-        });
-        if (isAlreadyAttached) {
-          logger2.info(`Database ${attachedDb.name} is already attached, skipping`);
-          continue;
-        }
-      } catch (error) {
-        logger2.warn(`Failed to check existing databases, proceeding with attachment:`, error);
-      }
-      switch (attachedDb.type) {
-        case "bigquery": {
-          if (!attachedDb.bigqueryConnection) {
-            throw new Error(`BigQuery connection configuration is missing for attached database: ${attachedDb.name}`);
-          }
-          await duckdbConnection.runSQL("INSTALL bigquery FROM community;");
-          await duckdbConnection.runSQL("LOAD bigquery;");
-          const bigqueryConfig = attachedDb.bigqueryConnection;
-          const attachParams = new URLSearchParams;
-          if (!bigqueryConfig.defaultProjectId) {
-            throw new Error(`BigQuery defaultProjectId is required for attached database: ${attachedDb.name}`);
-          }
-          attachParams.set("project", bigqueryConfig.defaultProjectId);
-          if (bigqueryConfig.serviceAccountKeyJson) {
-            const serviceAccountKeyPath = import_path2.default.join(TEMP_DIR_PATH, `duckdb-${attachedDb.name}-${v4_default()}-service-account-key.json`);
-            await import_promises.default.writeFile(serviceAccountKeyPath, bigqueryConfig.serviceAccountKeyJson);
-            attachParams.set("service_account_key", serviceAccountKeyPath);
-          }
-          const attachCommand = `ATTACH '${attachParams.toString()}' AS ${attachedDb.name} (TYPE bigquery, READ_ONLY);`;
-          try {
-            await duckdbConnection.runSQL(attachCommand);
-            logger2.info(`Successfully attached BigQuery database: ${attachedDb.name}`);
-          } catch (attachError) {
-            if (attachError instanceof Error && attachError.message && attachError.message.includes("already exists")) {
-              logger2.info(`BigQuery database ${attachedDb.name} is already attached, skipping`);
-            } else {
-              throw attachError;
-            }
-          }
-          break;
-        }
-        case "snowflake": {
-          if (!attachedDb.snowflakeConnection) {
-            throw new Error(`Snowflake connection configuration is missing for attached database: ${attachedDb.name}`);
-          }
-          await duckdbConnection.runSQL("INSTALL snowflake FROM community;");
-          await duckdbConnection.runSQL("LOAD snowflake;");
-          const snowflakeConfig = attachedDb.snowflakeConnection;
-          const attachParams = new URLSearchParams;
-          if (snowflakeConfig.account) {
-            attachParams.set("account", snowflakeConfig.account);
-          }
-          if (snowflakeConfig.username) {
-            attachParams.set("username", snowflakeConfig.username);
-          }
-          if (snowflakeConfig.password) {
-            attachParams.set("password", snowflakeConfig.password);
-          }
-          if (snowflakeConfig.database) {
-            attachParams.set("database", snowflakeConfig.database);
-          }
-          if (snowflakeConfig.warehouse) {
-            attachParams.set("warehouse", snowflakeConfig.warehouse);
-          }
-          if (snowflakeConfig.role) {
-            attachParams.set("role", snowflakeConfig.role);
-          }
-          const attachCommand = `ATTACH '${attachParams.toString()}' AS ${attachedDb.name} (TYPE snowflake, READ_ONLY);`;
-          try {
-            await duckdbConnection.runSQL(attachCommand);
-            logger2.info(`Successfully attached Snowflake database: ${attachedDb.name}`);
-          } catch (attachError) {
-            if (attachError instanceof Error && attachError.message && attachError.message.includes("already exists")) {
-              logger2.info(`Snowflake database ${attachedDb.name} is already attached, skipping`);
-            } else {
-              throw attachError;
-            }
-          }
-          break;
-        }
-        case "postgres": {
-          if (!attachedDb.postgresConnection) {
-            throw new Error(`PostgreSQL connection configuration is missing for attached database: ${attachedDb.name}`);
-          }
-          await duckdbConnection.runSQL("INSTALL postgres FROM community;");
-          await duckdbConnection.runSQL("LOAD postgres;");
-          const postgresConfig = attachedDb.postgresConnection;
-          let attachString;
-          if (postgresConfig.connectionString) {
-            attachString = postgresConfig.connectionString;
-          } else {
-            const params = new URLSearchParams;
-            if (postgresConfig.host) {
-              params.set("host", postgresConfig.host);
-            }
-            if (postgresConfig.port) {
-              params.set("port", postgresConfig.port.toString());
-            }
-            if (postgresConfig.databaseName) {
-              params.set("dbname", postgresConfig.databaseName);
-            }
-            if (postgresConfig.userName) {
-              params.set("user", postgresConfig.userName);
-            }
-            if (postgresConfig.password) {
-              params.set("password", postgresConfig.password);
-            }
-            attachString = params.toString();
-          }
-          const attachCommand = `ATTACH '${attachString}' AS ${attachedDb.name} (TYPE postgres, READ_ONLY);`;
-          try {
-            await duckdbConnection.runSQL(attachCommand);
-            logger2.info(`Successfully attached PostgreSQL database: ${attachedDb.name}`);
-          } catch (attachError) {
-            if (attachError instanceof Error && attachError.message && attachError.message.includes("already exists")) {
-              logger2.info(`PostgreSQL database ${attachedDb.name} is already attached, skipping`);
-            } else {
-              throw attachError;
-            }
-          }
-          break;
-        }
-        default:
-          throw new Error(`Unsupported attached database type: ${attachedDb.type}`);
+        await handler(duckdbConnection, attachedDb);
+      } catch (attachError) {
+        handleAlreadyAttachedError(attachError, attachedDb.name || "");
       }
     } catch (error) {
       logger2.error(`Failed to attach database ${attachedDb.name}:`, error);
@@ -132060,7 +132133,7 @@ async function attachDatabasesToDuckDB(duckdbConnection, attachedDatabases) {
     }
   }
 }
-async function createProjectConnections(connections = []) {
+async function createProjectConnections(connections = [], projectPath = "") {
   const connectionMap = new Map;
   const processedConnections = new Set;
   const apiConnections = [];
@@ -132177,6 +132250,15 @@ async function createProjectConnections(connections = []) {
         break;
       }
       case "duckdb": {
+        if (!connection.duckdbConnection) {
+          throw new Error("DuckDB connection configuration is missing.");
+        }
+        const duckdbConnection = new import_db_duckdb.DuckDBConnection(connection.name, ":memory:", projectPath);
+        if (connection.duckdbConnection.attachedDatabases && Array.isArray(connection.duckdbConnection.attachedDatabases) && connection.duckdbConnection.attachedDatabases.length > 0) {
+          await attachDatabasesToDuckDB(duckdbConnection, connection.duckdbConnection.attachedDatabases);
+        }
+        connectionMap.set(connection.name, duckdbConnection);
+        connection.attributes = getConnectionAttributes(duckdbConnection);
         break;
       }
       default: {
@@ -132255,6 +132337,12 @@ async function testConnectionConfig(connectionConfig) {
     if (!connectionConfig.name) {
       throw new Error("Connection name is required");
     }
+    if (connectionConfig.type === "duckdb") {
+      return {
+        status: "ok",
+        errorMessage: ""
+      };
+    }
     const { malloyConnections } = await createProjectConnections([connectionConfig]);
     const connection = malloyConnections.get(connectionConfig.name);
     if (!connection) {
@@ -132318,11 +132406,16 @@ async function getSchemasForConnection(connection, malloyConnection) {
     try {
       const bigquery = createBigQueryClient(connection);
       const [datasets] = await bigquery.getDatasets();
-      return datasets.map((dataset) => ({
-        name: dataset.id,
-        isHidden: false,
-        isDefault: false
+      const schemas = await Promise.all(datasets.map(async (dataset) => {
+        const [metadata] = await dataset.getMetadata();
+        return {
+          name: dataset.id,
+          isHidden: false,
+          isDefault: false,
+          description: metadata?.description
+        };
       }));
+      return schemas;
     } catch (error) {
       console.error(`Error getting schemas for BigQuery connection ${connection.name}:`, error);
       throw new Error(`Failed to get schemas for BigQuery connection ${connection.name}: ${error.message}`);
@@ -132409,20 +132502,24 @@ async function getSchemasForConnection(connection, malloyConnection) {
       throw new Error("DuckDB connection is required");
     }
     try {
-      const result = await malloyConnection.runSQL("SELECT DISTINCT schema_name FROM information_schema.schemata ORDER BY schema_name");
+      const result = await malloyConnection.runSQL("SELECT DISTINCT schema_name,catalog_name FROM information_schema.schemata ORDER BY catalog_name,schema_name", { rowLimit: 1000 });
       const rows = standardizeRunSQLResult(result);
-      const hasAttachedDatabases = connection.duckdbConnection?.attachedDatabases && Array.isArray(connection.duckdbConnection.attachedDatabases) && connection.duckdbConnection.attachedDatabases.length > 0;
       return rows.map((row) => {
         const typedRow = row;
-        let schemaName = typedRow.schema_name;
-        if (hasAttachedDatabases && schemaName !== "main") {
-          const attachedDbName = connection.duckdbConnection.attachedDatabases[0].name;
-          schemaName = `${attachedDbName}.${schemaName}`;
-        }
+        const schemaName = typedRow.schema_name;
+        const catalogName = typedRow.catalog_name;
         return {
-          name: schemaName,
-          isHidden: false,
-          isDefault: typedRow.schema_name === "main"
+          name: `${catalogName}.${schemaName}`,
+          isHidden: [
+            "information_schema",
+            "performance_schema",
+            "",
+            "SNOWFLAKE",
+            "information_schema",
+            "pg_catalog",
+            "pg_toast"
+          ].includes(schemaName) || ["md_information_schema", "system"].includes(catalogName),
+          isDefault: catalogName === "main"
         };
       });
     } catch (error) {
@@ -132568,15 +132665,9 @@ async function listTablesForSchema(connection, schemaName, malloyConnection) {
       throw new Error("DuckDB connection is required");
     }
     try {
-      const hasAttachedDatabases = connection.duckdbConnection?.attachedDatabases && Array.isArray(connection.duckdbConnection.attachedDatabases) && connection.duckdbConnection.attachedDatabases.length > 0;
-      let actualSchemaName = schemaName;
-      if (hasAttachedDatabases && schemaName.includes(".")) {
-        const attachedDbName = connection.duckdbConnection.attachedDatabases[0].name;
-        if (schemaName.startsWith(`${attachedDbName}.`)) {
-          actualSchemaName = schemaName.substring(attachedDbName.length + 1);
-        }
-      }
-      const result = await malloyConnection.runSQL(`SELECT table_name FROM information_schema.tables WHERE table_schema = '${actualSchemaName}' ORDER BY table_name`);
+      const catalogName = schemaName.split(".")[0];
+      schemaName = schemaName.split(".")[1];
+      const result = await malloyConnection.runSQL(`SELECT table_name FROM information_schema.tables WHERE table_schema = '${schemaName}' and table_catalog = '${catalogName}' ORDER BY table_name`, { rowLimit: 1000 });
       const rows = standardizeRunSQLResult(result);
       return rows.map((row) => {
         const typedRow = row;
@@ -132603,7 +132694,7 @@ class ConnectionController {
     if (connection.type === "duckdb") {
       const packages = await project.listPackages();
       if (packages.length === 0) {
-        throw new ConnectionError("No packages found for DuckDB connection");
+        return project.getMalloyConnection(connectionName);
       }
       const packageName = packages[0].name;
       if (!packageName) {
@@ -132657,7 +132748,8 @@ class ConnectionController {
     const tableSource = await getConnectionTableSource(malloyConnection, tablePath.split(".").pop(), tablePath);
     return {
       resource: tablePath,
-      columns: tableSource.columns
+      columns: tableSource.columns,
+      source: tableSource.source
     };
   }
   async getConnectionQueryData(projectName, connectionName, sqlStatement, options) {
@@ -139347,7 +139439,7 @@ class Project {
     }
     if (payload.connections) {
       logger2.info(`Updating ${payload.connections.length} connections for project ${this.projectName}`);
-      const { malloyConnections, apiConnections } = await createProjectConnections(payload.connections);
+      const { malloyConnections, apiConnections } = await createProjectConnections(payload.connections, this.projectPath);
       this.malloyConnections = malloyConnections;
       this.apiConnections = apiConnections;
       logger2.info(`Successfully updated connections for project ${this.projectName}`, {
@@ -139363,7 +139455,7 @@ class Project {
       throw new ProjectNotFoundError(`Project path ${projectPath} not found`);
     }
     logger2.info(`Creating project with connection configuration`);
-    const { malloyConnections, apiConnections } = await createProjectConnections(connections);
+    const { malloyConnections, apiConnections } = await createProjectConnections(connections, projectPath);
     logger2.info(`Loaded ${malloyConnections.size + apiConnections.length} connections for project ${projectName}`, {
       malloyConnections,
       apiConnections

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@malloy-publisher/server",
   "description": "Malloy Publisher Server",
-  "version": "0.0.122",
+  "version": "0.0.124",
   "main": "dist/server.js",
   "bin": {
     "malloy-publisher": "dist/server.js"

package/src/controller/connection.controller.ts

@@ -40,9 +40,7 @@ export class ConnectionController {
       if (connection.type === "duckdb") {
          const packages = await project.listPackages();
          if (packages.length === 0) {
-            throw new ConnectionError(
-               "No packages found for DuckDB connection",
-            );
+            return project.getMalloyConnection(connectionName);
          }
          // For now, use the first package's DuckDB connection
          const packageName = packages[0].name;
@@ -165,6 +163,7 @@ export class ConnectionController {
       return {
          resource: tablePath,
          columns: tableSource.columns,
+         source: tableSource.source,
       };
    }