@malloy-publisher/server 0.0.165 → 0.0.167

package/src/service/connection.ts

@@ -858,6 +858,103 @@ function getConnectionAttributes(
    };
 }
 
+async function testDuckDBConnection(
+   duckdbConnection: DuckDBConnection,
+   connectionConfig: InternalConnection,
+): Promise<void> {
+   // Test base DuckDB connection with a simple query
+   try {
+      await duckdbConnection.runSQL("SELECT 1 AS test");
+      logger.info(
+         `DuckDB base connection test passed for: ${connectionConfig.name}`,
+      );
+   } catch (error) {
+      throw new Error(
+         `DuckDB base connection test failed: ${(error as Error).message}`,
+      );
+   }
+
+   // Test each attached database if configured
+   const attachedDatabases =
+      connectionConfig.duckdbConnection?.attachedDatabases;
+   if (!attachedDatabases || attachedDatabases.length === 0) {
+      return;
+   }
+
+   const failedAttachments: string[] = [];
+
+   for (const attachedDb of attachedDatabases) {
+      if (!attachedDb.name) {
+         continue;
+      }
+
+      try {
+         // Test the attached database by querying its tables/schemas
+         // Different database types require different test queries
+         switch (attachedDb.type) {
+            case "postgres": {
+               // Test postgres attachment by listing schemas
+               await duckdbConnection.runSQL(
+                  `SELECT schema_name FROM information_schema.schemata WHERE catalog_name = '${attachedDb.name}' LIMIT 1`,
+               );
+               logger.info(
+                  `Attached Postgres database test passed: ${attachedDb.name}`,
+               );
+               break;
+            }
+            case "bigquery": {
+               // Test BigQuery attachment by listing datasets
+               // BigQuery attached databases show as catalogs
+               await duckdbConnection.runSQL(
+                  `SELECT database_name FROM duckdb_databases() WHERE database_name = '${attachedDb.name}'`,
+               );
+               logger.info(
+                  `Attached BigQuery database test passed: ${attachedDb.name}`,
+               );
+               break;
+            }
+            case "snowflake": {
+               // Test Snowflake attachment by verifying database is attached
+               await duckdbConnection.runSQL(
+                  `SELECT database_name FROM duckdb_databases() WHERE database_name = '${attachedDb.name}'`,
+               );
+               logger.info(
+                  `Attached Snowflake database test passed: ${attachedDb.name}`,
+               );
+               break;
+            }
+            case "gcs":
+            case "s3": {
+               // For cloud storage, verify the secret was created
+               // Cloud storage doesn't attach as a database, it uses secrets for auth
+               await duckdbConnection.runSQL(
+                  `SELECT name FROM duckdb_secrets() WHERE name LIKE '%${attachedDb.name}%' LIMIT 1`,
+               );
+               logger.info(
+                  `Cloud storage credentials test passed: ${attachedDb.name}`,
+               );
+               break;
+            }
+            default: {
+               logger.warn(
+                  `Unknown attached database type: ${attachedDb.type}`,
+               );
+            }
+         }
+      } catch (error) {
+         const errorMessage = `Attached database '${attachedDb.name}' (${attachedDb.type}) test failed: ${(error as Error).message}`;
+         logger.error(errorMessage);
+         failedAttachments.push(errorMessage);
+      }
+   }
+
+   if (failedAttachments.length > 0) {
+      throw new Error(
+         `DuckDB connection test failed for attached databases:\n${failedAttachments.join("\n")}`,
+      );
+   }
+}
+
 export async function testConnectionConfig(
    connectionConfig: ApiConnection,
 ): Promise<ApiConnectionStatus> {
@@ -868,8 +965,6 @@ export async function testConnectionConfig(
       }
 
       // Use createProjectConnections to create the connection, then test it
-      // TODO: Test duckdb connections?
-
       const { malloyConnections } = await createProjectConnections(
          [connectionConfig], // Pass the single connection config
       );
@@ -882,16 +977,23 @@ export async function testConnectionConfig(
          );
       }
 
-      // Test the connection - cast to union type of connection classes that have test method
-      await (
-         connection as
-            | PostgresConnection
-            | BigQueryConnection
-            | SnowflakeConnection
-            | TrinoConnection
-            | MySQLConnection
-            | DuckDBConnection
-      ).test();
+      // Handle DuckDB connections specially since they have attached databases
+      if (connectionConfig.type === "duckdb") {
+         await testDuckDBConnection(
+            connection as DuckDBConnection,
+            connectionConfig as InternalConnection,
+         );
+      } else {
+         // Test other connection types using their test() method
+         await (
+            connection as
+               | PostgresConnection
+               | BigQueryConnection
+               | SnowflakeConnection
+               | TrinoConnection
+               | MySQLConnection
+         ).test();
+      }
 
       return {
          status: "ok",

package/src/service/db_utils.ts

@@ -7,9 +7,8 @@ import {
    CloudStorageCredentials,
    gcsConnectionToCredentials,
    getCloudTablesWithColumns,
-   listAllDataFilesInBucket,
-   listCloudBuckets,
-   listFilesInCloudDirectory,
+   listCloudDirectorySchemas,
+   listDataFilesInDirectory,
    parseCloudUri,
    s3ConnectionToCredentials,
 } from "./gcs_s3_utils";
@@ -340,23 +339,10 @@ export async function getSchemasForConnection(
                   : s3ConnectionToCredentials(attachedDb.s3Connection!);
 
             try {
-               const buckets = await listCloudBuckets(credentials);
-               const scheme = dbType === "gcs" ? "gs" : "s3";
-
-               logger.info(
-                  `Listed ${buckets.length} ${dbType.toUpperCase()} buckets for attached database ${attachedDb.name}`,
-               );
-
-               // Just return bucket URIs as schemas - fast!
-               // Files/directories will be listed when user selects a bucket
-               return buckets.map((bucket) => ({
-                  name: `${scheme}://${bucket.name}`,
-                  isHidden: false,
-                  isDefault: false,
-               }));
+               return await listCloudDirectorySchemas(credentials);
             } catch (cloudError) {
                logger.warn(
-                  `Failed to list ${dbType.toUpperCase()} buckets for ${attachedDb.name}`,
+                  `Failed to list ${dbType.toUpperCase()} directory schemas for ${attachedDb.name}`,
                   { error: cloudError },
                );
                return [];
@@ -444,19 +430,11 @@ export async function getTablesForSchema(
          );
       }
 
-      let fileKeys: string[];
-      if (directoryPath) {
-         const fileNames = await listFilesInCloudDirectory(
-            credentials,
-            bucketName,
-            directoryPath,
-         );
-         fileKeys = fileNames.map((fileName) => `${directoryPath}/${fileName}`);
-      } else {
-         fileKeys = await listAllDataFilesInBucket(credentials, bucketName);
-      }
-
-      console.log("File keys:", fileKeys);
+      const fileKeys = await listDataFilesInDirectory(
+         credentials,
+         bucketName,
+         directoryPath,
+      );
 
       return await getCloudTablesWithColumns(
          malloyConnection,
@@ -575,7 +553,7 @@ export async function getConnectionTableSource(
             type: field.type,
          };
       });
-      logger.info(`Successfully fetched schema for ${tablePath}`, {
+      logger.debug(`Successfully fetched schema for ${tablePath}`, {
          fieldCount: fields.length,
       });
       return {
@@ -748,15 +726,15 @@ export async function listTablesForSchema(
          }
 
          try {
-            if (directoryPath) {
-               return await listFilesInCloudDirectory(
-                  credentials,
-                  bucketName,
-                  directoryPath,
-               );
-            } else {
-               return await listAllDataFilesInBucket(credentials, bucketName);
-            }
+            const fileKeys = await listDataFilesInDirectory(
+               credentials,
+               bucketName,
+               directoryPath,
+            );
+            return fileKeys.map((key) => {
+               const lastSlash = key.lastIndexOf("/");
+               return lastSlash > 0 ? key.substring(lastSlash + 1) : key;
+            });
          } catch (error) {
             logger.error(
                `Error listing ${cloudType.toUpperCase()} objects in ${schemaName}`,

package/src/service/gcs_s3_utils.ts

@@ -8,7 +8,6 @@ import { components } from "../api";
 import { logger } from "../logger";
 
 type ApiTable = components["schemas"]["Table"];
-
 type CloudStorageType = "gcs" | "s3";
 
 export interface CloudStorageCredentials {
@@ -29,7 +28,6 @@ interface CloudStorageObject {
    key: string;
    size?: number;
    lastModified?: Date;
-   isFolder: boolean;
 }
 
 export function gcsConnectionToCredentials(gcsConnection: {
@@ -92,7 +90,7 @@ function createCloudStorageClient(
    return client;
 }
 
-export async function listCloudBuckets(
+async function listCloudBuckets(
    credentials: CloudStorageCredentials,
 ): Promise<CloudStorageBucket[]> {
    const client = createCloudStorageClient(credentials);
@@ -143,7 +141,6 @@ async function listAllCloudFiles(
                   key: content.Key,
                   size: content.Size,
                   lastModified: content.LastModified,
-                  isFolder: false,
                });
             }
          }
@@ -182,6 +179,15 @@ function isDataFile(key: string): boolean {
    );
 }
 
+function buildCloudUri(
+   type: CloudStorageType,
+   bucket: string,
+   key: string,
+): string {
+   const scheme = type === "gcs" ? "gs" : "s3";
+   return `${scheme}://${bucket}/${key}`;
+}
+
 function getFileType(key: string): string {
    const lowerKey = key.toLowerCase();
    if (lowerKey.endsWith(".csv")) return "csv";
@@ -192,23 +198,14 @@ function getFileType(key: string): string {
    return "unknown";
 }
 
-function buildCloudUri(
-   type: CloudStorageType,
-   bucket: string,
-   key: string,
-): string {
-   const scheme = type === "gcs" ? "gs" : "s3";
-   return `${scheme}://${bucket}/${key}`;
-}
-
 function standardizeRunSQLResult(result: unknown): unknown[] {
    return Array.isArray(result)
       ? result
       : (result as { rows?: unknown[] }).rows || [];
 }
 
-// Batch size for parallel schema fetching to avoid overwhelming the connection
 const SCHEMA_FETCH_BATCH_SIZE = 10;
+const BUCKET_SCAN_BATCH_SIZE = 3;
 
 async function getTableSchema(
    malloyConnection: Connection,
@@ -268,11 +265,9 @@ export async function getCloudTablesWithColumns(
 ): Promise<ApiTable[]> {
    const allTables: ApiTable[] = [];
 
-   // Process in batches to avoid overwhelming the connection
    for (let i = 0; i < fileKeys.length; i += SCHEMA_FETCH_BATCH_SIZE) {
       const batch = fileKeys.slice(i, i + SCHEMA_FETCH_BATCH_SIZE);
 
-      // Process batch in parallel
       const batchResults = await Promise.all(
          batch.map((fileKey) =>
             getTableSchema(malloyConnection, credentials, bucketName, fileKey),
@@ -315,38 +310,118 @@ export function parseCloudUri(uri: string): {
    return null;
 }
 
-export async function listFilesInCloudDirectory(
+export async function listDataFilesInDirectory(
    credentials: CloudStorageCredentials,
    bucketName: string,
    directoryPath: string,
 ): Promise<string[]> {
-   const files = await listAllCloudFiles(credentials, bucketName);
-
-   const filesInDirectory = files
-      .filter((obj) => {
-         if (!isDataFile(obj.key)) return false;
-
-         const lastSlashIndex = obj.key.lastIndexOf("/");
-         const fileDir =
-            lastSlashIndex > 0 ? obj.key.substring(0, lastSlashIndex) : "";
-
-         return fileDir === directoryPath;
-      })
-      .map((obj) => {
-         const lastSlashIndex = obj.key.lastIndexOf("/");
-         return lastSlashIndex > 0
-            ? obj.key.substring(lastSlashIndex + 1)
-            : obj.key;
-      });
+   const prefix = directoryPath ? `${directoryPath}/` : "";
+   const client = createCloudStorageClient(credentials);
+   const storageType = credentials.type.toUpperCase();
+   const dataFiles: string[] = [];
+
+   try {
+      let continuationToken: string | undefined;
+
+      do {
+         const response = await client.send(
+            new ListObjectsV2Command({
+               Bucket: bucketName,
+               Prefix: prefix,
+               Delimiter: "/",
+               ContinuationToken: continuationToken,
+            }),
+         );
+
+         for (const content of response.Contents || []) {
+            if (content.Key && isDataFile(content.Key)) {
+               dataFiles.push(content.Key);
+            }
+         }
 
-   return filesInDirectory;
+         continuationToken = response.IsTruncated
+            ? response.NextContinuationToken
+            : undefined;
+      } while (continuationToken);
+
+      logger.info(
+         `Listed ${dataFiles.length} data files in ${storageType} ${bucketName}/${directoryPath}`,
+      );
+      return dataFiles;
+   } catch (error) {
+      logger.error(
+         `Failed to list files in ${storageType} ${bucketName}/${directoryPath}`,
+         { error },
+      );
+      throw new Error(
+         `Failed to list files in ${storageType} ${bucketName}/${directoryPath}: ${error instanceof Error ? error.message : String(error)}`,
+      );
+   }
 }
 
-// List all data files in a bucket with their full relative paths
-export async function listAllDataFilesInBucket(
+/**
+ * Scans an entire bucket and returns unique directory paths that contain data files.
+ * Uses flat listing for efficiency — O(total_files / 1000) API calls.
+ */
+async function listDirectorySchemas(
    credentials: CloudStorageCredentials,
    bucketName: string,
 ): Promise<string[]> {
-   const files = await listAllCloudFiles(credentials, bucketName);
-   return files.filter((obj) => isDataFile(obj.key)).map((obj) => obj.key);
+   const allFiles = await listAllCloudFiles(credentials, bucketName);
+   const directories = new Set<string>();
+
+   for (const file of allFiles) {
+      if (!isDataFile(file.key)) continue;
+
+      const lastSlashIndex = file.key.lastIndexOf("/");
+      const dir =
+         lastSlashIndex > 0 ? file.key.substring(0, lastSlashIndex) : "";
+      directories.add(dir);
+   }
+
+   const scheme = credentials.type === "gcs" ? "gs" : "s3";
+   const sortedDirs = Array.from(directories).sort();
+
+   logger.info(
+      `Found ${sortedDirs.length} directories with data files in ${credentials.type.toUpperCase()} bucket ${bucketName}`,
+   );
+
+   return sortedDirs.map((dir) =>
+      dir ? `${scheme}://${bucketName}/${dir}` : `${scheme}://${bucketName}`,
+   );
+}
+
+export async function listCloudDirectorySchemas(
+   credentials: CloudStorageCredentials,
+): Promise<{ name: string; isHidden: boolean; isDefault: boolean }[]> {
+   const storageType = credentials.type.toUpperCase();
+   const buckets = await listCloudBuckets(credentials);
+
+   logger.info(
+      `Listed ${buckets.length} ${storageType} buckets, scanning for directories...`,
+   );
+
+   const allDirArrays: string[][] = [];
+
+   for (let i = 0; i < buckets.length; i += BUCKET_SCAN_BATCH_SIZE) {
+      const batch = buckets.slice(i, i + BUCKET_SCAN_BATCH_SIZE);
+      const batchResults = await Promise.all(
+         batch.map((bucket) =>
+            listDirectorySchemas(credentials, bucket.name).catch((err) => {
+               logger.warn(
+                  `Failed to scan ${storageType} bucket ${bucket.name}`,
+                  { error: err },
+               );
+               return [] as string[];
+            }),
+         ),
+      );
+      allDirArrays.push(...batchResults);
+   }
+
+   return allDirArrays.flat().map((dirUri) => ({
+      name: dirUri,
+      isHidden: false,
+      isDefault: false,
+   }));
 }

package/src/service/model.ts

@@ -211,13 +211,13 @@ export class Model {
       } catch (error) {
          let computedError = error;
          if (error instanceof Error && error.stack) {
-            console.error("Error stack", error.stack);
+            logger.error("Error stack", error.stack);
          }
 
          if (error instanceof MalloyError) {
             const problems = error.problems;
             for (const problem of problems) {
-               console.error("Problem", problem);
+               logger.error("Problem", problem);
             }
             computedError = new ModelCompilationError(error);
          }
@@ -444,7 +444,7 @@ export class Model {
       const notebookCells: ApiNotebookCell[] = (
          this.runnableNotebookCells as RunnableNotebookCell[]
       ).map((cell) => {
-         console.log("cell.queryInfo", cell.queryInfo);
+         logger.debug("cell.queryInfo", cell.queryInfo);
          return {
             type: cell.type,
             text: cell.text,
@@ -550,9 +550,9 @@ export class Model {
                   text: cell.text,
                };
             } else {
-               console.log("Error message: ", errorMessage);
+               logger.error("Error message: ", errorMessage);
             }
-            console.log("Cell content: ", cellIndex, cell.type, cell.text);
+            logger.debug("Cell content: ", cellIndex, cell.type, cell.text);
             throw new BadRequestError(`Cell execution failed: ${errorMessage}`);
          }
       }

package/src/service/project.ts

@@ -1,3 +1,5 @@
+import type { LogMessage } from "@malloydata/malloy";
+import { FixedConnectionMap, MalloyError, Runtime } from "@malloydata/malloy";
 import { BaseConnection } from "@malloydata/malloy/connection";
 import { Mutex } from "async-mutex";
 import * as fs from "fs";
@@ -10,6 +12,7 @@ import {
    ProjectNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
+import { URL_READER } from "../utils";
 import { createProjectConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -159,6 +162,71 @@ export class Project {
       return this.metadata;
    }
 
+   public async compileSource(
+      packageName: string,
+      modelName: string,
+      source: string,
+      includeSql: boolean = false,
+   ): Promise<{ problems: LogMessage[]; sql?: string }> {
+      // Place the virtual file in the model's directory so relative imports resolve correctly.
+      const modelDir = path.dirname(
+         path.join(this.projectPath, packageName, modelName),
+      );
+      const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
+      const virtualUrl = new URL(virtualUri);
+
+      // Read the model file and extract its preamble (pragmas + imports) so that
+      // the user's query inherits the model's import context.
+      const modelPath = path.join(this.projectPath, packageName, modelName);
+      const preamble = await extractPreamble(modelPath);
+      const fullSource = preamble ? `${preamble}\n${source}` : source;
+
+      // Create a URL Reader that serves the source string for the virtual file,
+      // but falls back to the disk for everything else (imports).
+      const interceptingReader = {
+         readURL: async (url: URL) => {
+            if (url.toString() === virtualUri) {
+               return fullSource;
+            }
+            return URL_READER.readURL(url);
+         },
+      };
+
+      // Initialize Runtime with the project's active connections
+      const runtime = new Runtime({
+         urlReader: interceptingReader,
+         connections: new FixedConnectionMap(this.malloyConnections, "duckdb"),
+      });
+
+      // Attempt to compile
+      try {
+         const modelMaterializer = runtime.loadModel(virtualUrl);
+         const model = await modelMaterializer.getModel();
+
+         // If includeSql is requested and compilation succeeded, attempt to extract SQL
+         let sql: string | undefined;
+         if (includeSql) {
+            try {
+               const queryMaterializer = modelMaterializer.loadFinalQuery();
+               sql = await queryMaterializer.getSQL();
+            } catch {
+               // Source may not contain a runnable query (e.g. only source definitions),
+               // in which case we simply omit the sql field.
+            }
+         }
+
+         // If successful, return any non-fatal warnings
+         return { problems: model.problems, sql };
+      } catch (error) {
+         // If parsing/compilation fails, return the errors
+         if (error instanceof MalloyError) {
+            return { problems: error.problems };
+         }
+         // If it's a system error (e.g. file not found), throw it up
+         throw error;
+      }
+   }
+
    public listApiConnections(): ApiConnection[] {
       return this.apiConnections;
    }
@@ -244,11 +312,18 @@ export class Project {
       // package multiple times.
       let packageMutex = this.packageMutexes.get(packageName);
       if (packageMutex?.isLocked()) {
+         logger.debug(
+            `Package ${packageName} is being loaded, waiting for unlock...`,
+         );
          await packageMutex.waitForUnlock();
+         logger.debug(`Package ${packageName} unlocked`);
          const existingPackage = this.packages.get(packageName);
          if (existingPackage) {
+            logger.debug(`Package ${packageName} loaded by another request`);
             return existingPackage;
          }
+         // If package still doesn't exist after unlock, it might have failed to load
+         // Continue to try loading it ourselves
       }
       packageMutex = new Mutex();
       this.packageMutexes.set(packageName, packageMutex);
@@ -264,19 +339,23 @@ export class Project {
          this.setPackageStatus(packageName, PackageStatus.LOADING);
 
          try {
+            logger.debug(`Loading package ${packageName}...`);
+            const packagePath = path.join(this.projectPath, packageName);
             const _package = await Package.create(
                this.projectName,
                packageName,
-               path.join(this.projectPath, packageName),
+               packagePath,
                this.malloyConnections,
             );
             this.packages.set(packageName, _package);
 
             // Set package status to serving
             this.setPackageStatus(packageName, PackageStatus.SERVING);
+            logger.debug(`Successfully loaded package ${packageName}`);
 
             return _package;
          } catch (error) {
+            logger.error(`Failed to load package ${packageName}`, { error });
             // Clean up on error - mutex will be automatically released by runExclusive
             this.packages.delete(packageName);
             this.packageStatuses.delete(packageName);
@@ -537,3 +616,43 @@ export class Project {
          });
    }
 }
+
+/**
+ * Extracts the preamble from a Malloy model file — the leading block of
+ * `##!` pragmas, `import` statements, blank lines, and comments that appear
+ * before any `source:`, `query:`, or `run:` definition. This allows a
+ * submitted query to inherit the model's import context.
+ */
+export async function extractPreamble(modelPath: string): Promise<string> {
+   try {
+      const content = await fs.promises.readFile(modelPath, "utf8");
+      return extractPreambleFromSource(content);
+   } catch {
+      // If the model file can't be read, return empty preamble
+      // and let the compilation surface any import errors naturally.
+      return "";
+   }
+}
+
+/**
+ * Extracts the preamble from Malloy source text. Exported for testing.
+ */
+export function extractPreambleFromSource(content: string): string {
+   const lines = content.split("\n");
+   const preambleLines: string[] = [];
+
+   for (const line of lines) {
+      const trimmed = line.trim();
+      // Stop at the first source/query/run definition
+      if (
+         trimmed.startsWith("source:") ||
+         trimmed.startsWith("query:") ||
+         trimmed.startsWith("run:")
+      ) {
+         break;
+      }
+      preambleLines.push(line);
+   }
+
+   return preambleLines.join("\n").trimEnd();
+}