@malloy-publisher/server 0.0.170 → 0.0.172

package/src/service/db_utils.ts

@@ -1,3 +1,5 @@
+import { ClientSecretCredential } from "@azure/identity";
+import { ContainerClient } from "@azure/storage-blob";
 import { BigQuery } from "@google-cloud/bigquery";
 import { Connection, TableSourceDef } from "@malloydata/malloy";
 import { components } from "../api";
@@ -17,6 +19,7 @@ import { ApiConnection } from "./model";
 type ApiSchema = components["schemas"]["Schema"];
 type ApiTable = components["schemas"]["Table"];
 type ApiTableSource = components["schemas"]["TableSource"];
+type ApiAzureConnection = components["schemas"]["AzureConnection"];
 
 function createBigQueryClient(connection: ApiConnection): BigQuery {
    if (!connection.bigqueryConnection) {
@@ -354,6 +357,21 @@ export async function getSchemasForConnection(
             schemas.push(...cloudSchemas);
          }
 
+         // Add Azure ADLS attached databases as schemas (by name)
+         const azureDatabases = attachedDatabases.filter(
+            (attachedDb) =>
+               attachedDb.type === "azure" && attachedDb.azureConnection,
+         );
+         for (const attachedDb of azureDatabases) {
+            if (attachedDb.name) {
+               schemas.push({
+                  name: attachedDb.name,
+                  isHidden: false,
+                  isDefault: false,
+               });
+            }
+         }
+
          return schemas;
       } catch (error) {
          console.error(
@@ -434,11 +452,281 @@ export async function getSchemasForConnection(
    }
 }
 
+function getFileType(key: string): string {
+   const lowerKey = key.toLowerCase();
+   if (lowerKey.endsWith(".csv")) return "csv";
+   if (lowerKey.endsWith(".parquet")) return "parquet";
+   if (lowerKey.endsWith(".json")) return "json";
+   if (lowerKey.endsWith(".jsonl") || lowerKey.endsWith(".ndjson"))
+      return "jsonl";
+   return "unknown";
+}
+
+/**
+ * Lists blobs in an Azure container matching a glob-like prefix/extension filter.
+ * Parses an HTTPS SAS URL like:
+ *   https://account.blob.core.windows.net/container/path/*.parquet?sasToken
+ * Returns individual file URLs with the SAS token appended.
+ */
+async function listAzureBlobs(
+   fileUrl: string,
+   azureConnection?: ApiAzureConnection,
+): Promise<{ url: string; blobName: string }[]> {
+   // Split URL and SAS token carefully to avoid encoding issues with signatures
+   const queryStart = fileUrl.indexOf("?");
+   const baseUrl = queryStart >= 0 ? fileUrl.substring(0, queryStart) : fileUrl;
+   const sasToken = queryStart >= 0 ? fileUrl.substring(queryStart) : "";
+
+   // Parse the URL to extract account, container, and blob path
+   let accountUrl: string;
+   let container: string;
+   let blobPath: string;
+
+   if (baseUrl.startsWith("abfss://")) {
+      // abfss://container/path or abfss://account.dfs.core.windows.net/container/path
+      const withoutScheme = baseUrl.substring("abfss://".length);
+      const parts = withoutScheme.split("/").filter(Boolean);
+      if (parts[0].includes(".")) {
+         // Fully qualified: abfss://account.dfs.core.windows.net/container/path
+         const accountName = parts[0].split(".")[0];
+         accountUrl = `https://${accountName}.blob.core.windows.net`;
+         container = parts[1];
+         blobPath = parts.slice(2).join("/");
+      } else {
+         // Short form: abfss://container/path — need accountName from config
+         if (!azureConnection?.accountName) {
+            throw new Error(
+               "accountName is required to list blobs with abfss:// URLs",
+            );
+         }
+         accountUrl = `https://${azureConnection.accountName}.blob.core.windows.net`;
+         container = parts[0];
+         blobPath = parts.slice(1).join("/");
+      }
+   } else {
+      // https://account.blob.core.windows.net/container/path
+      const url = new URL(baseUrl);
+      const pathParts = url.pathname.split("/").filter(Boolean);
+      container = pathParts[0];
+      blobPath = pathParts.slice(1).join("/");
+      accountUrl = `${url.protocol}//${url.host}`;
+   }
+
+   // Three supported glob patterns:
+   //   path/file.ext       → single file (handled upstream by isAzureSingleFileUrl)
+   //   path/*.ext          → files directly in path/ with that extension (no subdirs)
+   //   path/**             → all valid data files in path/ and nested dirs (recursive)
+   let prefix: string;
+   let extensionFilter = ""; // for *.ext pattern
+   let recursive = true; // for ** pattern
+
+   if (blobPath.endsWith("**")) {
+      // Recursive listing: everything under this prefix
+      prefix = blobPath.slice(0, -2);
+      recursive = true;
+   } else if (blobPath.includes("*")) {
+      // Single-level glob: path/*.ext — files directly in that dir only
+      const starIndex = blobPath.indexOf("*");
+      prefix = blobPath.substring(0, starIndex);
+      extensionFilter = blobPath.substring(starIndex + 1); // e.g. ".parquet"
+      recursive = false;
+   } else {
+      // No glob — use blobPath as prefix (container-level listing)
+      prefix = blobPath;
+      recursive = true;
+   }
+
+   // Create ContainerClient with appropriate authentication
+   let containerClient: ContainerClient;
+   if (
+      azureConnection?.authType === "service_principal" &&
+      azureConnection.tenantId &&
+      azureConnection.clientId &&
+      azureConnection.clientSecret
+   ) {
+      const credential = new ClientSecretCredential(
+         azureConnection.tenantId,
+         azureConnection.clientId,
+         azureConnection.clientSecret,
+      );
+      containerClient = new ContainerClient(
+         `${accountUrl}/${container}`,
+         credential,
+      );
+   } else {
+      // SAS token auth — append token to container URL
+      const containerUrl = `${accountUrl}/${container}${sasToken}`;
+      containerClient = new ContainerClient(containerUrl);
+   }
+
+   const matchingFiles: { url: string; blobName: string }[] = [];
+   for await (const blob of containerClient.listBlobsFlat({
+      prefix: prefix || undefined,
+   })) {
+      if (extensionFilter && !blob.name.endsWith(extensionFilter)) continue;
+      // For *.ext (non-recursive): only allow files directly in prefix dir
+      if (!recursive) {
+         const nameAfterPrefix = blob.name.substring(prefix.length);
+         if (nameAfterPrefix.includes("/")) continue;
+      }
+      if (!isDataFile(blob.name)) continue;
+      // For SPN: use abfss:// URLs that DuckDB's azure extension can read
+      // For SAS: use https:// URLs with token appended
+      let url: string;
+      if (azureConnection?.authType === "service_principal") {
+         const account =
+            azureConnection.accountName ||
+            accountUrl.split("//")[1]?.split(".")[0];
+         url = `abfss://${account}.dfs.core.windows.net/${container}/${blob.name}`;
+      } else {
+         url = `${accountUrl}/${container}/${blob.name}${sasToken}`;
+      }
+      matchingFiles.push({ url, blobName: blob.name });
+   }
+
+   logger.info(
+      `Listed ${matchingFiles.length} matching blobs in Azure container ${container} with prefix "${prefix}"`,
+   );
+   return matchingFiles;
+}
+
+function isDataFile(key: string): boolean {
+   const lowerKey = key.toLowerCase();
+   return (
+      lowerKey.endsWith(".csv") ||
+      lowerKey.endsWith(".parquet") ||
+      lowerKey.endsWith(".json") ||
+      lowerKey.endsWith(".jsonl") ||
+      lowerKey.endsWith(".ndjson")
+   );
+}
+
+async function describeRemoteFile(
+   malloyConnection: Connection,
+   fileUri: string,
+): Promise<ApiTable> {
+   const pathWithoutQuery = fileUri.split("?")[0];
+   const fileType = getFileType(pathWithoutQuery);
+
+   let describeQuery: string;
+   switch (fileType) {
+      case "csv":
+         describeQuery = `DESCRIBE SELECT * FROM read_csv('${fileUri}', auto_detect=true) LIMIT 1`;
+         break;
+      case "parquet":
+         describeQuery = `DESCRIBE SELECT * FROM read_parquet('${fileUri}') LIMIT 1`;
+         break;
+      case "json":
+         describeQuery = `DESCRIBE SELECT * FROM read_json('${fileUri}', auto_detect=true) LIMIT 1`;
+         break;
+      case "jsonl":
+         describeQuery = `DESCRIBE SELECT * FROM read_json('${fileUri}', format='newline_delimited', auto_detect=true) LIMIT 1`;
+         break;
+      default:
+         logger.warn(`Unsupported file type for file: ${fileUri}`);
+         return { resource: fileUri, columns: [] };
+   }
+
+   const result = await malloyConnection.runSQL(describeQuery);
+   const rows = standardizeRunSQLResult(result);
+   const columns = rows.map((row: unknown) => {
+      const typedRow = row as Record<string, unknown>;
+      return {
+         name: (typedRow.column_name || typedRow.name) as string,
+         type: (typedRow.column_type || typedRow.type) as string,
+      };
+   });
+
+   const fileName = pathWithoutQuery.split("/").pop() || fileUri;
+   return { resource: fileName, columns };
+}
+
+function isAzureSingleFileUrl(fileUri: string): boolean {
+   const pathWithoutQuery = fileUri.split("?")[0];
+   // Has a glob — not a single file
+   if (pathWithoutQuery.includes("*")) return false;
+   // Ends with / — directory listing
+   if (pathWithoutQuery.endsWith("/")) return false;
+   // Check if the last path segment has a data file extension
+   const lastSegment = pathWithoutQuery.split("/").pop() || "";
+   return isDataFile(lastSegment);
+}
+
+async function describeAzureFile(
+   malloyConnection: Connection,
+   fileUri: string,
+   azureConnection?: ApiAzureConnection,
+): Promise<ApiTable[]> {
+   try {
+      if (isAzureSingleFileUrl(fileUri)) {
+         // Single file — describe directly via DuckDB
+         return [await describeRemoteFile(malloyConnection, fileUri)];
+      }
+
+      // Glob pattern or container/directory URL — list blobs via Azure SDK
+      const blobs = await listAzureBlobs(fileUri, azureConnection);
+      if (blobs.length === 0) {
+         return [{ resource: fileUri, columns: [] }];
+      }
+
+      const results = await Promise.all(
+         blobs.map(async ({ url, blobName }) => {
+            try {
+               const table = await describeRemoteFile(malloyConnection, url);
+               return { ...table, resource: blobName };
+            } catch (error) {
+               logger.warn(`Failed to describe Azure blob: ${url}`, { error });
+               return { resource: blobName, columns: [] } as ApiTable;
+            }
+         }),
+      );
+      return results;
+   } catch (error) {
+      logger.error(`Failed to describe Azure file: ${fileUri}`, { error });
+      throw new Error(
+         `Failed to describe Azure file: ${error instanceof Error ? error.message : String(error)}`,
+      );
+   }
+}
+
 export async function getTablesForSchema(
    connection: ApiConnection,
    schemaName: string,
    malloyConnection: Connection,
 ): Promise<ApiTable[]> {
+   // Check if schemaName matches an Azure attached database name
+   if (connection.type === "duckdb") {
+      const attachedDbs = connection.duckdbConnection?.attachedDatabases || [];
+      const azureDb = attachedDbs.find(
+         (db) =>
+            db.type === "azure" && db.name === schemaName && db.azureConnection,
+      );
+      if (azureDb) {
+         const azureConn = azureDb.azureConnection!;
+         const fileUrl =
+            azureConn.authType === "sas_token"
+               ? azureConn.sasUrl
+               : azureConn.fileUrl;
+         if (fileUrl) {
+            return await describeAzureFile(
+               malloyConnection,
+               fileUrl,
+               azureConn,
+            );
+         }
+      }
+   }
+
+   // Check if this is an Azure ADLS file path (abfss:// or HTTPS SAS URL)
+   if (
+      connection.type === "duckdb" &&
+      (schemaName.startsWith("abfss://") ||
+         schemaName.startsWith("https://") ||
+         schemaName.startsWith("az://"))
+   ) {
+      return await describeAzureFile(malloyConnection, schemaName);
+   }
+
    // Check if this is a cloud storage file path (gs://bucket/path/file.ext or s3://bucket/path/file.ext)
    const parsedUri = parseCloudUri(schemaName);
 

package/src/service/project_store.ts

@@ -20,6 +20,7 @@ import {
    PUBLISHER_DATA_DIR,
 } from "../constants";
 import {
+   BadRequestError,
    FrozenConfigError,
    PackageNotFoundError,
    ProjectNotFoundError,
@@ -31,6 +32,63 @@ import { StorageConfig, StorageManager } from "../storage/StorageManager";
 import { PackageStatus, Project } from "./project";
 type ApiProject = components["schemas"]["Project"];
 
+const AZURE_SUPPORTED_SCHEMES = ["https://", "http://", "abfss://", "az://"];
+const AZURE_DATA_EXTENSIONS = [
+   ".parquet",
+   ".csv",
+   ".json",
+   ".jsonl",
+   ".ndjson",
+];
+
+function validateAzureUrl(url: string, fieldName: string): void {
+   if (!AZURE_SUPPORTED_SCHEMES.some((s) => url.startsWith(s))) {
+      throw new BadRequestError(
+         `Azure ${fieldName} must use one of: ${AZURE_SUPPORTED_SCHEMES.join(", ")}`,
+      );
+   }
+   const pathWithoutQuery = url.split("?")[0];
+   const stars = (pathWithoutQuery.match(/\*/g) || []).length;
+
+   if (stars === 0) {
+      const lower = pathWithoutQuery.toLowerCase();
+      if (!AZURE_DATA_EXTENSIONS.some((ext) => lower.endsWith(ext))) {
+         throw new BadRequestError(
+            `Azure ${fieldName}: a single-file URL must end with a data file extension (${AZURE_DATA_EXTENSIONS.join(", ")})`,
+         );
+      }
+   } else if (pathWithoutQuery.endsWith("**")) {
+      // recursive — valid
+      // includes all data files in the container and all subdirectories
+   } else {
+      const lastSegment = pathWithoutQuery.split("/").pop() || "";
+      if (stars !== 1 || !lastSegment.startsWith("*")) {
+         throw new BadRequestError(
+            `Azure ${fieldName}: only three URL patterns are supported:\n` +
+               `  • Single file:    path/file.parquet\n` +
+               `  • Directory glob: path/*.ext  (direct children only)\n` +
+               `  • Recursive:      path/**     (includes all data files in the container and all subdirectories)\n` +
+               `Multi-level globs such as "sub_dir/*/*.parquet" are not supported.`,
+         );
+      }
+   }
+}
+
+function validateProjectAzureUrls(project: ApiProject): void {
+   for (const conn of project.connections || []) {
+      if (conn.type !== "duckdb") continue;
+      for (const db of conn.duckdbConnection?.attachedDatabases || []) {
+         if (db.type !== "azure" || !db.azureConnection) continue;
+         const { authType, sasUrl, fileUrl } = db.azureConnection;
+         if (authType === "sas_token" && sasUrl) {
+            validateAzureUrl(sasUrl, `"${db.name}" sasUrl`);
+         } else if (authType === "service_principal" && fileUrl) {
+            validateAzureUrl(fileUrl, `"${db.name}" fileUrl`);
+         }
+      }
+   }
+}
+
 export class ProjectStore {
    public serverRootPath: string;
    private projects: Map<string, Project> = new Map();
@@ -762,6 +820,7 @@ export class ProjectStore {
       if (this.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
+      validateProjectAzureUrls(project);
       const projectName = project.name;
       if (!projectName) {
          throw new Error("Project name is required");
@@ -1112,7 +1171,12 @@ export class ProjectStore {
             logger.info(
                `Downloading S3 directory from "${location}" to "${targetPath}"`,
             );
-            await this.downloadS3Directory(location, projectName, targetPath);
+            await this.downloadS3Directory(
+               location,
+               projectName,
+               targetPath,
+               isCompressedFile,
+            );
             return;
          } catch (error) {
             const errorData = this.extractErrorDataFromError(error);
@@ -1242,10 +1306,43 @@ export class ProjectStore {
       s3Path: string,
       projectName: string,
       absoluteDirPath: string,
+      isCompressedFile: boolean = false,
    ) {
       const trimmedPath = s3Path.slice(5);
       const [bucketName, ...prefixParts] = trimmedPath.split("/");
       const prefix = prefixParts.join("/");
+
+      if (isCompressedFile) {
+         // Download the single zip file
+         const zipFilePath = `${absoluteDirPath}.zip`;
+         await fs.promises.mkdir(path.dirname(zipFilePath), {
+            recursive: true,
+         });
+
+         const command = new GetObjectCommand({
+            Bucket: bucketName,
+            Key: prefix,
+         });
+         const item = await this.s3Client.send(command);
+         if (!item.Body) {
+            throw new ProjectNotFoundError(
+               `Project ${projectName} not found in ${s3Path}`,
+            );
+         }
+         const file = fs.createWriteStream(zipFilePath);
+         item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
+         await new Promise<void>((resolve, reject) => {
+            file.on("error", reject);
+            file.on("finish", resolve);
+         });
+
+         // Extract the zip file
+         await this.unzipProject(zipFilePath);
+         logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
+         return;
+      }
+
+      // Original behavior: download directory contents
       const objects = await this.s3Client.listObjectsV2({
          Bucket: bucketName,
          Prefix: prefix,
@@ -1291,6 +1388,7 @@ export class ProjectStore {
             });
          }),
       );
+      logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
    }
 
    private parseGitHubUrl(

package/dist/app/assets/HomePage-oXJcPThQ.js

@@ -1 +0,0 @@
-import{l as o,j as r,r as a}from"./index-DUmIwFQ_.js";function s(){const t=o();return r.jsx(a,{onClickProject:t})}export{s as default};

package/dist/app/assets/ModelPage-DtywEBbr.js

@@ -1 +0,0 @@
-import{t as n,j as e,D as i,E as t,F as c}from"./index-DUmIwFQ_.js";function o(){const a=n(),r=a["*"];if(!a.projectName)return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})});if(!a.packageName)return e.jsx("div",{children:e.jsx("h2",{children:"Missing package name"})});const s=i({projectName:a.projectName,packageName:a.packageName,modelPath:r});return r?.endsWith(".malloy")?e.jsx(t,{resourceUri:s,runOnDemand:!0,maxResultSize:512*1024}):r?.endsWith(".malloynb")?e.jsx(c,{resourceUri:s,maxResultSize:1024*1024}):e.jsx("div",{children:e.jsxs("h2",{children:["Unrecognized file type: ",r]})})}export{o as default};

package/dist/app/assets/ProjectPage-CvmjaWsE.js

@@ -1 +0,0 @@
-import{l as a,t as n,j as e,D as c,H as o}from"./index-DUmIwFQ_.js";function j(){const r=a(),{projectName:s}=n();if(s){const t=c({projectName:s});return e.jsx(o,{onSelectPackage:r,resourceUri:t})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})})}export{j as default};

package/dist/app/assets/WorkbookPage-NNNUEWM5.js

@@ -1 +0,0 @@
-import{t as a,j as e,D as t,K as c}from"./index-DUmIwFQ_.js";function d(){const{workspace:r,workbookPath:s,projectName:i,packageName:n}=a();if(r)if(s)if(i)if(n){const o=t({projectName:i,packageName:n});return e.jsx(c,{workbookPath:{path:s,workspace:r},resourceUri:o},`${s}`)}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing package name"})});else return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})});else return e.jsx("div",{children:e.jsx("h2",{children:"Missing workbook path"})});else return e.jsx("div",{children:e.jsx("h2",{children:"Missing workspace"})})}export{d as default};