@malloy-publisher/server 0.0.171 → 0.0.173

package/src/controller/connection.controller.ts

@@ -20,6 +20,71 @@ type ApiTable = components["schemas"]["Table"];
 type ApiQueryData = components["schemas"]["QueryData"];
 type ApiTemporaryTable = components["schemas"]["TemporaryTable"];
 type ApiSchema = components["schemas"]["Schema"];
+const AZURE_SUPPORTED_SCHEMES = ["https://", "http://", "abfss://", "az://"];
+const AZURE_DATA_EXTENSIONS = [
+   ".parquet",
+   ".csv",
+   ".json",
+   ".jsonl",
+   ".ndjson",
+];
+
+/**
+ * Validates an Azure URL against the three supported patterns:
+ *   1. Single file:         path/file.parquet
+ *   2. Directory glob:      path/*.ext   (direct children only, no sub-dirs)
+ *   3. Recursive:           path/**      (all data files recursively)
+ */
+function validateAzureUrl(url: string, fieldName: string): void {
+   if (!AZURE_SUPPORTED_SCHEMES.some((s) => url.startsWith(s))) {
+      throw new BadRequestError(
+         `Azure ${fieldName} must use one of: ${AZURE_SUPPORTED_SCHEMES.join(", ")}`,
+      );
+   }
+
+   const pathWithoutQuery = url.split("?")[0];
+   const stars = (pathWithoutQuery.match(/\*/g) || []).length;
+
+   if (stars === 0) {
+      // Single file — must end with a data extension
+      const lower = pathWithoutQuery.toLowerCase();
+      if (!AZURE_DATA_EXTENSIONS.some((ext) => lower.endsWith(ext))) {
+         throw new BadRequestError(
+            `Azure ${fieldName}: a single-file URL must end with a data file extension (${AZURE_DATA_EXTENSIONS.join(", ")})`,
+         );
+      }
+   } else if (pathWithoutQuery.endsWith("**")) {
+      // Recursive — valid, no further checks needed
+   } else {
+      // Must be exactly path/*.ext — one star, in the last path segment only
+      const lastSegment = pathWithoutQuery.split("/").pop() || "";
+      if (stars !== 1 || !lastSegment.startsWith("*")) {
+         throw new BadRequestError(
+            `Azure ${fieldName}: only three URL patterns are supported:\n` +
+               `  • Single file:      path/file.parquet\n` +
+               `  • Directory glob:   path/*.ext  (direct children only)\n` +
+               `  • Recursive:        path/**     (all data files in subtree)\n` +
+               `Multi-level globs such as "sub_dir/*/*.parquet" are not supported.`,
+         );
+      }
+   }
+}
+
+function validateAzureAttachedDatabases(connectionConfig: ApiConnection): void {
+   if (connectionConfig.type !== "duckdb") return;
+   const attachedDbs =
+      connectionConfig.duckdbConnection?.attachedDatabases || [];
+   for (const db of attachedDbs) {
+      if (db.type !== "azure" || !db.azureConnection) continue;
+      const { authType, sasUrl, fileUrl } = db.azureConnection;
+      if (authType === "sas_token" && sasUrl) {
+         validateAzureUrl(sasUrl, `"${db.name}" sasUrl`);
+      } else if (authType === "service_principal" && fileUrl) {
+         validateAzureUrl(fileUrl, `"${db.name}" fileUrl`);
+      }
+   }
+}
+
 export class ConnectionController {
    private projectStore: ProjectStore;
    private connectionService: ConnectionService;
@@ -162,7 +227,9 @@ export class ConnectionController {
          projectName,
          connectionName,
       );
-      const connection = await this.getConnection(projectName, connectionName);
+      // Use getApiConnection to get the unwrapped ApiConnection config, consistent with listSchemas and listTables.
+      const project = await this.projectStore.getProject(projectName, false);
+      const connection = project.getApiConnection(connectionName);
 
       if (connection.type === "ducklake") {
          if (tablePath.split(".").length === 1) {
@@ -178,6 +245,53 @@ export class ConnectionController {
          // If tablePath already has 3+ parts or starts with connection name, use as-is
       }
 
+      // Check if this is an Azure attached database
+      if (connection.type === "duckdb") {
+         const attachedDbs =
+            connection.duckdbConnection?.attachedDatabases || [];
+         const azureDb = attachedDbs.find(
+            (db) =>
+               db.type === "azure" &&
+               db.name === schemaName &&
+               db.azureConnection,
+         );
+         if (azureDb && azureDb.azureConnection) {
+            // Reconstruct the full SAS URL for the specific file
+            const azureConn = azureDb.azureConnection;
+            const baseUrl =
+               azureConn.authType === "sas_token"
+                  ? azureConn.sasUrl
+                  : azureConn.fileUrl;
+            if (baseUrl) {
+               // Extract the file name from tablePath (e.g., "a.aircraft.parquet" -> "aircraft.parquet")
+               const fileName = tablePath.includes(".")
+                  ? tablePath.split(".").slice(1).join(".")
+                  : tablePath;
+               // Replace the file portion in the base URL with the specific file name
+               const urlParts = baseUrl.split("?");
+               const basePath = urlParts[0];
+               const queryString = urlParts[1] ? `?${urlParts[1]}` : "";
+               // Replace the last path segment (or glob) with the actual file name
+               const dirPath = basePath.substring(
+                  0,
+                  basePath.lastIndexOf("/") + 1,
+               );
+               const fullFileUrl = `${dirPath}${fileName}${queryString}`;
+
+               const tableSource = await getConnectionTableSource(
+                  malloyConnection,
+                  fileName,
+                  fullFileUrl,
+               );
+               return {
+                  resource: tablePath,
+                  columns: tableSource.columns,
+                  source: tableSource.source,
+               };
+            }
+         }
+      }
+
       const tableKey = tablePath.split(".").pop();
       if (!tableKey) {
          throw new Error(`Invalid tablePath: ${tablePath}`);
@@ -254,6 +368,16 @@ export class ConnectionController {
    public async testConnectionConfiguration(
       connectionConfig: ApiConnection,
    ): Promise<ApiConnectionStatus> {
+      if (
+         connectionConfig &&
+         "config" in connectionConfig &&
+         typeof (connectionConfig as Record<string, unknown>).config ===
+            "object"
+      ) {
+         connectionConfig = (connectionConfig as Record<string, unknown>)
+            .config as ApiConnection;
+      }
+
       if (
          !connectionConfig ||
          typeof connectionConfig !== "object" ||
@@ -297,6 +421,8 @@ export class ConnectionController {
          throw new BadRequestError("Connection type is required");
       }
 
+      validateAzureAttachedDatabases(connectionConfig);
+
       logger.info(
          `Creating connection "${connectionName}" in project "${projectName}"`,
       );
@@ -321,6 +447,8 @@ export class ConnectionController {
          throw new BadRequestError("Connection payload is required");
       }
 
+      validateAzureAttachedDatabases(connection as ApiConnection);
+
       logger.info(
          `Updating connection "${connectionName}" in project "${projectName}"`,
       );

package/src/controller/watch-mode.controller.ts

@@ -22,8 +22,8 @@ export class WatchModeController {
    public getWatchStatus: Handler<void, WatchStatusRes> = async (_req, res) => {
       return res.json({
          enabled: !!this.watchingPath,
-         watchingPath: this.watchingPath,
-         projectName: this.watchingProjectName ?? undefined,
+         watchingPath: this.watchingPath ?? "",
+         projectName: this.watchingProjectName ?? "",
       });
    };
 

package/src/service/connection.ts

@@ -589,6 +589,78 @@ async function attachCloudStorage(
    logger.info(`${storageType} connection configured for: ${attachedDb.name}`);
 }
 
+async function attachAzureStorage(
+   connection: DuckDBConnection,
+   attachedDb: AttachedDatabase,
+): Promise<void> {
+   if (!attachedDb.azureConnection) {
+      throw new Error(
+         `Azure connection configuration missing for: ${attachedDb.name}`,
+      );
+   }
+
+   const config = attachedDb.azureConnection;
+
+   // Extensions are loaded once in attachDatabasesToDuckDB before the loop
+   const secretName = sanitizeSecretName(`azure_${attachedDb.name}`);
+
+   let createSecretCommand: string;
+
+   if (config.authType === "service_principal") {
+      if (
+         !config.tenantId ||
+         !config.clientId ||
+         !config.clientSecret ||
+         !config.accountName
+      ) {
+         throw new Error(
+            `Azure SPN auth requires tenantId, clientId, clientSecret, and accountName for: ${attachedDb.name}`,
+         );
+      }
+
+      const escapedTenantId = escapeSQL(config.tenantId);
+      const escapedClientId = escapeSQL(config.clientId);
+      const escapedClientSecret = escapeSQL(config.clientSecret);
+      const escapedAccountName = escapeSQL(config.accountName);
+
+      createSecretCommand = `
+         CREATE OR REPLACE SECRET ${secretName} (
+            TYPE azure,
+            PROVIDER service_principal,
+            TENANT_ID '${escapedTenantId}',
+            CLIENT_ID '${escapedClientId}',
+            CLIENT_SECRET '${escapedClientSecret}',
+            ACCOUNT_NAME '${escapedAccountName}'
+         );
+      `;
+   } else if (config.authType === "sas_token") {
+      if (!config.sasUrl) {
+         throw new Error(
+            `Azure SAS token auth requires sasUrl for: ${attachedDb.name}`,
+         );
+      }
+
+      // For SAS token auth, DuckDB can read the HTTPS SAS URL directly via httpfs.
+      // No Azure secret is needed — just ensure httpfs is loaded.
+      logger.info(
+         `Azure SAS token configured for: ${attachedDb.name} (no secret needed, using direct URL)`,
+      );
+      return;
+   } else {
+      throw new Error(
+         `Unsupported Azure auth type: ${config.authType} for: ${attachedDb.name}`,
+      );
+   }
+
+   if (await doesSecretExistInDuckDB(connection, secretName)) {
+      await connection.runSQL(`DETACH ${attachedDb.name};`).catch(() => {});
+   }
+   await connection.runSQL(createSecretCommand);
+
+   logger.info(`Created Azure secret: ${secretName}`);
+   logger.info(`Azure ADLS connection configured for: ${attachedDb.name}`);
+}
+
 async function doesSecretExistInDuckDB(
    connection: DuckDBConnection,
    secretName: string,
@@ -614,8 +686,16 @@ async function attachDatabasesToDuckDB(
       postgres: attachPostgres,
       gcs: attachCloudStorage,
       s3: attachCloudStorage,
+      azure: attachAzureStorage,
    };
 
+   // Pre-load extensions needed by any attached database type, once per connection
+   const hasAzure = attachedDatabases.some((db) => db.type === "azure");
+   if (hasAzure) {
+      await installAndLoadExtension(duckdbConnection, "azure");
+      await installAndLoadExtension(duckdbConnection, "httpfs");
+   }
+
    for (const attachedDb of attachedDatabases) {
       try {
          // Check if already attached
@@ -650,6 +730,106 @@ async function attachDatabasesToDuckDB(
    }
 }
 
+type ApiAzureConnection = components["schemas"]["AzureConnection"];
+
+/**
+ * Builds the actual Azure URL for a blob given an AzureConnection config.
+ * Strips any glob pattern from the base URL and appends the blob name.
+ */
+function buildAzureFileUrl(
+   azureConn: ApiAzureConnection,
+   blobName: string,
+): string {
+   if (azureConn.authType === "sas_token" && azureConn.sasUrl) {
+      const qIdx = azureConn.sasUrl.indexOf("?");
+      const baseUrl =
+         qIdx >= 0 ? azureConn.sasUrl.substring(0, qIdx) : azureConn.sasUrl;
+      const token = qIdx >= 0 ? azureConn.sasUrl.substring(qIdx) : "";
+      // Single file URL — replace the filename portion so we don't double-append
+      if (/\.(parquet|csv|json|jsonl|ndjson)$/i.test(baseUrl)) {
+         const dir = baseUrl.replace(/\/[^/]+$/, "");
+         return `${dir}/${blobName}${token}`;
+      }
+      // Glob or directory — strip trailing glob/slash and append blobName
+      const cleanBase = baseUrl.replace(/\/\*[^/]*$/, "").replace(/\/+$/, "");
+      return `${cleanBase}/${blobName}${token}`;
+   } else if (azureConn.authType === "service_principal" && azureConn.fileUrl) {
+      const url = azureConn.fileUrl;
+      // Single file URL (ends with a data file extension) — replace the
+      // filename with blobName so we don't double-append
+      if (/\.(parquet|csv|json|jsonl|ndjson)$/i.test(url)) {
+         return url.replace(/\/[^/]+$/, `/${blobName}`);
+      }
+      // Glob or directory — strip glob pattern and trailing slash, append blobName
+      const base = url.replace(/\*[^/]*$/, "").replace(/\/+$/, "");
+      return `${base}/${blobName}`;
+   }
+   throw new Error(
+      `Cannot build Azure file URL: missing sasUrl or fileUrl in config`,
+   );
+}
+
+/**
+ * Extends DuckDBConnection to resolve Azure attached-database table paths.
+ * When Malloy compiles di.table('azure_schema.blob_name'), the path
+ * 'azure_schema.blob_name' is passed to fetchTableSchema.  This override
+ * detects that prefix, constructs the real Azure URL, and forwards it to
+ * DuckDB so the azure/httpfs extension can read the file.
+ */
+class AzureDuckDBConnection extends DuckDBConnection {
+   private azureDatabases: AttachedDatabase[];
+
+   constructor(
+      connectionName: string,
+      databasePath: string,
+      workingDirectory: string,
+      azureDatabases: AttachedDatabase[],
+   ) {
+      super(connectionName, databasePath, workingDirectory);
+      this.azureDatabases = azureDatabases;
+   }
+
+   async fetchTableSchema(
+      tableKey: string,
+      tablePath: string,
+   ): Promise<TableSourceDef> {
+      const dotIdx = tablePath.indexOf(".");
+      if (dotIdx > 0) {
+         const schemaName = tablePath.substring(0, dotIdx);
+         const blobName = tablePath.substring(dotIdx + 1);
+
+         const azureDb = this.azureDatabases.find(
+            (db) =>
+               db.type === "azure" &&
+               db.name === schemaName &&
+               db.azureConnection,
+         );
+
+         if (azureDb) {
+            const azureUrl = buildAzureFileUrl(
+               azureDb.azureConnection!,
+               blobName,
+            );
+            logger.debug("Resolved Azure table path", {
+               original: tablePath,
+               resolved: azureUrl,
+            });
+            const result = await super.fetchTableSchema(tableKey, azureUrl);
+            if (!result) {
+               throw new Error(`Azure file not found: ${azureUrl}`);
+            }
+            return result;
+         }
+      }
+
+      const result = await super.fetchTableSchema(tableKey, tablePath);
+      if (!result) {
+         throw new Error(`Table ${tablePath} not found`);
+      }
+      return result;
+   }
+}
+
 class DuckLakeConnection extends DuckDBConnection {
    private connectionName: string;
 
@@ -968,21 +1148,29 @@ export async function createProjectConnections(
             // Create DuckDB connection with project basePath as working directory
             // This ensures relative paths in the project are resolved correctly
             // Use unique memory database path to prevent sharing across connections
-            const duckdbConnection = new DuckDBConnection(
-               connection.name,
-               path.join(projectPath, `${connection.name}.duckdb`),
-               projectPath,
+            const attachedDatabases =
+               connection.duckdbConnection.attachedDatabases ?? [];
+            const hasAzureAttached = attachedDatabases.some(
+               (db) => db.type === "azure",
             );
+            const duckdbConnection = hasAzureAttached
+               ? new AzureDuckDBConnection(
+                    connection.name,
+                    path.join(projectPath, `${connection.name}.duckdb`),
+                    projectPath,
+                    attachedDatabases,
+                 )
+               : new DuckDBConnection(
+                    connection.name,
+                    path.join(projectPath, `${connection.name}.duckdb`),
+                    projectPath,
+                 );
 
             // Attach databases if configured
-            if (
-               connection.duckdbConnection.attachedDatabases &&
-               Array.isArray(connection.duckdbConnection.attachedDatabases) &&
-               connection.duckdbConnection.attachedDatabases.length > 0
-            ) {
+            if (attachedDatabases.length > 0) {
                await attachDatabasesToDuckDB(
                   duckdbConnection,
-                  connection.duckdbConnection.attachedDatabases,
+                  attachedDatabases,
                );
             }
 
@@ -1088,6 +1276,7 @@ function getConnectionAttributes(
    };
 }
 
+// TODO: Re-write these tests to validate based on credentials provided in the connection config
 async function testDuckDBConnection(
    duckdbConnection: DuckDBConnection,
    connectionConfig: InternalConnection,
@@ -1153,10 +1342,16 @@ async function testDuckDBConnection(
                );
                break;
             }
-            case "gcs":
+            case "gcs": {
+               await duckdbConnection.runSQL(
+                  `SELECT name FROM duckdb_secrets() WHERE name LIKE '%${attachedDb.name}%' LIMIT 1`,
+               );
+               logger.info(
+                  `Cloud storage credentials test passed: ${attachedDb.name}`,
+               );
+               break;
+            }
             case "s3": {
-               // For cloud storage, verify the secret was created
-               // Cloud storage doesn't attach as a database, it uses secrets for auth
                await duckdbConnection.runSQL(
                   `SELECT name FROM duckdb_secrets() WHERE name LIKE '%${attachedDb.name}%' LIMIT 1`,
                );
@@ -1165,6 +1360,28 @@ async function testDuckDBConnection(
                );
                break;
             }
+            case "azure": {
+               const azureConfig = attachedDb.azureConnection;
+               if (azureConfig?.authType === "sas_token") {
+                  // SAS token is embedded in the URL — no DuckDB secret is created.
+                  if (!azureConfig.sasUrl) {
+                     throw new Error(
+                        `Azure SAS token URL is missing for: ${attachedDb.name}`,
+                     );
+                  }
+                  logger.info(
+                     `Azure SAS token URL present for: ${attachedDb.name}`,
+                  );
+               } else {
+                  await duckdbConnection.runSQL(
+                     `SELECT name FROM duckdb_secrets() WHERE name LIKE '%${attachedDb.name}%' LIMIT 1`,
+                  );
+                  logger.info(
+                     `Azure SPN credentials test passed: ${attachedDb.name}`,
+                  );
+               }
+               break;
+            }
             default: {
                logger.warn(
                   `Unknown attached database type: ${attachedDb.type}`,