@malloy-publisher/server 0.0.165 → 0.0.168

package/src/service/project.ts

@@ -1,3 +1,5 @@
+import type { LogMessage } from "@malloydata/malloy";
+import { FixedConnectionMap, MalloyError, Runtime } from "@malloydata/malloy";
 import { BaseConnection } from "@malloydata/malloy/connection";
 import { Mutex } from "async-mutex";
 import * as fs from "fs";
@@ -10,7 +12,12 @@ import {
    ProjectNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
-import { createProjectConnections, InternalConnection } from "./connection";
+import { URL_READER } from "../utils";
+import {
+   createProjectConnections,
+   deleteDuckLakeConnectionFile,
+   InternalConnection,
+} from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
 
@@ -83,12 +90,13 @@ export class Project {
          logger.info(
             `Updating ${payload.connections.length} connections for project ${this.projectName}`,
          );
-
+         const isUpdateConnectionRequest = true;
          // Reload connections with full config
          const { malloyConnections, apiConnections } =
             await createProjectConnections(
                payload.connections,
                this.projectPath,
+               isUpdateConnectionRequest,
             );
 
          // Update the project's connection maps
@@ -159,6 +167,71 @@ export class Project {
       return this.metadata;
    }
 
+   public async compileSource(
+      packageName: string,
+      modelName: string,
+      source: string,
+      includeSql: boolean = false,
+   ): Promise<{ problems: LogMessage[]; sql?: string }> {
+      // Place the virtual file in the model's directory so relative imports resolve correctly.
+      const modelDir = path.dirname(
+         path.join(this.projectPath, packageName, modelName),
+      );
+      const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
+      const virtualUrl = new URL(virtualUri);
+
+      // Read the model file and extract its preamble (pragmas + imports) so that
+      // the user's query inherits the model's import context.
+      const modelPath = path.join(this.projectPath, packageName, modelName);
+      const preamble = await extractPreamble(modelPath);
+      const fullSource = preamble ? `${preamble}\n${source}` : source;
+
+      // Create a URL Reader that serves the source string for the virtual file,
+      // but falls back to the disk for everything else (imports).
+      const interceptingReader = {
+         readURL: async (url: URL) => {
+            if (url.toString() === virtualUri) {
+               return fullSource;
+            }
+            return URL_READER.readURL(url);
+         },
+      };
+
+      // Initialize Runtime with the project's active connections
+      const runtime = new Runtime({
+         urlReader: interceptingReader,
+         connections: new FixedConnectionMap(this.malloyConnections, "duckdb"),
+      });
+
+      // Attempt to compile
+      try {
+         const modelMaterializer = runtime.loadModel(virtualUrl);
+         const model = await modelMaterializer.getModel();
+
+         // If includeSql is requested and compilation succeeded, attempt to extract SQL
+         let sql: string | undefined;
+         if (includeSql) {
+            try {
+               const queryMaterializer = modelMaterializer.loadFinalQuery();
+               sql = await queryMaterializer.getSQL();
+            } catch {
+               // Source may not contain a runnable query (e.g. only source definitions),
+               // in which case we simply omit the sql field.
+            }
+         }
+
+         // If successful, return any non-fatal warnings
+         return { problems: model.problems, sql };
+      } catch (error) {
+         // If parsing/compilation fails, return the errors
+         if (error instanceof MalloyError) {
+            return { problems: error.problems };
+         }
+         // If it's a system error (e.g. file not found), throw it up
+         throw error;
+      }
+   }
+
    public listApiConnections(): ApiConnection[] {
       return this.apiConnections;
    }
@@ -244,11 +317,18 @@ export class Project {
       // package multiple times.
       let packageMutex = this.packageMutexes.get(packageName);
       if (packageMutex?.isLocked()) {
+         logger.debug(
+            `Package ${packageName} is being loaded, waiting for unlock...`,
+         );
          await packageMutex.waitForUnlock();
+         logger.debug(`Package ${packageName} unlocked`);
          const existingPackage = this.packages.get(packageName);
          if (existingPackage) {
+            logger.debug(`Package ${packageName} loaded by another request`);
             return existingPackage;
          }
+         // If package still doesn't exist after unlock, it might have failed to load
+         // Continue to try loading it ourselves
       }
       packageMutex = new Mutex();
       this.packageMutexes.set(packageName, packageMutex);
@@ -264,19 +344,23 @@ export class Project {
          this.setPackageStatus(packageName, PackageStatus.LOADING);
 
          try {
+            logger.debug(`Loading package ${packageName}...`);
+            const packagePath = path.join(this.projectPath, packageName);
             const _package = await Package.create(
                this.projectName,
                packageName,
-               path.join(this.projectPath, packageName),
+               packagePath,
                this.malloyConnections,
             );
             this.packages.set(packageName, _package);
 
             // Set package status to serving
             this.setPackageStatus(packageName, PackageStatus.SERVING);
+            logger.debug(`Successfully loaded package ${packageName}`);
 
             return _package;
          } catch (error) {
+            logger.error(`Failed to load package ${packageName}`, { error });
             // Clean up on error - mutex will be automatically released by runExclusive
             this.packages.delete(packageName);
             this.packageStatuses.delete(packageName);
@@ -457,8 +541,11 @@ export class Project {
          (conn) => conn.name === connectionName,
       );
 
-      if (this.apiConnections[index]?.type === "duckdb") {
+      const connectionType = this.apiConnections[index]?.type;
+      if (connectionType === "duckdb") {
          await this.deleteDuckDBConnection(connectionName);
+      } else if (connectionType === "ducklake") {
+         await this.deleteDuckLakeConnection(connectionName);
       }
 
       if (index !== -1) {
@@ -536,4 +623,53 @@ export class Project {
             );
          });
    }
+
+   public async deleteDuckLakeConnection(
+      connectionName: string,
+   ): Promise<void> {
+      await deleteDuckLakeConnectionFile(connectionName, this.projectPath);
+      logger.info(
+         `Removed DuckLake connection ${connectionName} from project ${this.projectName}`,
+      );
+   }
+}
+
+/**
+ * Extracts the preamble from a Malloy model file — the leading block of
+ * `##!` pragmas, `import` statements, blank lines, and comments that appear
+ * before any `source:`, `query:`, or `run:` definition. This allows a
+ * submitted query to inherit the model's import context.
+ */
+export async function extractPreamble(modelPath: string): Promise<string> {
+   try {
+      const content = await fs.promises.readFile(modelPath, "utf8");
+      return extractPreambleFromSource(content);
+   } catch {
+      // If the model file can't be read, return empty preamble
+      // and let the compilation surface any import errors naturally.
+      return "";
+   }
+}
+
+/**
+ * Extracts the preamble from Malloy source text. Exported for testing.
+ */
+export function extractPreambleFromSource(content: string): string {
+   const lines = content.split("\n");
+   const preambleLines: string[] = [];
+
+   for (const line of lines) {
+      const trimmed = line.trim();
+      // Stop at the first source/query/run definition
+      if (
+         trimmed.startsWith("source:") ||
+         trimmed.startsWith("query:") ||
+         trimmed.startsWith("run:")
+      ) {
+         break;
+      }
+      preambleLines.push(line);
+   }
+
+   return preambleLines.join("\n").trimEnd();
 }

package/src/service/project_compile.spec.ts

@@ -0,0 +1,197 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs";
+import path from "path";
+import { extractPreamble, extractPreambleFromSource } from "./project";
+
+describe("extractPreambleFromSource", () => {
+   it("should extract pragmas and imports before a source definition", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import "utils.malloy"',
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+         '  dimension: name is "test"',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import "utils.malloy"',
+            'import { revenue } from "metrics.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should extract pragmas and imports before a run statement", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "run: my_source -> { aggregate: count() }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should extract pragmas and imports before a query definition", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "query: top_items is my_source -> { limit: 10 }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should return empty string when file starts with source:", () => {
+      const content = 'source: my_source is duckdb.table("data.parquet")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with run:", () => {
+      const content = 'run: duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with query:", () => {
+      const content = 'query: q is duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should preserve comments in the preamble", () => {
+      const content = [
+         "##! experimental.parameters",
+         "// This model defines revenue metrics",
+         'import { revenue } from "metrics.malloy"',
+         "// Main source below",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            "// This model defines revenue metrics",
+            'import { revenue } from "metrics.malloy"',
+            "// Main source below",
+         ].join("\n"),
+      );
+   });
+
+   it("should handle multiple import blocks", () => {
+      const content = [
+         'import { a } from "file_a.malloy"',
+         'import { b } from "file_b.malloy"',
+         'import { c, d } from "file_c.malloy"',
+         "",
+         "source: combined is a {",
+         "  join_one: b on b.id = a.b_id",
+         "}",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            'import { a } from "file_a.malloy"',
+            'import { b } from "file_b.malloy"',
+            'import { c, d } from "file_c.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should return all content when there are no source/query/run definitions", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import { revenue } from "metrics.malloy"',
+         "",
+         "// no definitions yet",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import { revenue } from "metrics.malloy"',
+            "",
+            "// no definitions yet",
+         ].join("\n"),
+      );
+   });
+
+   it("should return empty string for empty input", () => {
+      expect(extractPreambleFromSource("")).toBe("");
+   });
+
+   it("should handle indented source/query/run definitions", () => {
+      const content = [
+         'import "model.malloy"',
+         "",
+         '  source: indented is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      // Indented definitions should still be detected (trimmed before check)
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import "model.malloy"');
+   });
+
+   it("should not stop on 'source' appearing in comments or strings", () => {
+      const content = [
+         "// This file defines the source of truth",
+         'import "model.malloy"',
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "// This file defines the source of truth",
+            'import "model.malloy"',
+         ].join("\n"),
+      );
+   });
+});
+
+describe("extractPreamble (file-based)", () => {
+   const testDir = path.join(process.cwd(), "test-temp-preamble");
+   const testModelPath = path.join(testDir, "test_model.malloy");
+
+   beforeEach(() => {
+      if (!fs.existsSync(testDir)) {
+         fs.mkdirSync(testDir, { recursive: true });
+      }
+   });
+
+   afterEach(() => {
+      if (fs.existsSync(testDir)) {
+         fs.rmSync(testDir, { recursive: true, force: true });
+      }
+   });
+
+   it("should read and extract preamble from a file", async () => {
+      const content = [
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+      fs.writeFileSync(testModelPath, content);
+
+      const result = await extractPreamble(testModelPath);
+      expect(result).toBe('import { revenue } from "metrics.malloy"');
+   });
+
+   it("should return empty string when file does not exist", async () => {
+      const result = await extractPreamble(
+         path.join(testDir, "nonexistent.malloy"),
+      );
+      expect(result).toBe("");
+   });
+});

package/src/service/project_store.ts

@@ -196,7 +196,8 @@ export class ProjectStore {
          );
       } catch (error) {
          markNotReady();
-         logger.error("Error initializing project store", { error });
+         const errorData = this.extractErrorDataFromError(error);
+         logger.error("Error initializing project store", errorData);
          process.exit(1);
       }
    }
@@ -533,7 +534,7 @@ export class ProjectStore {
             PUBLISHER_DATA_DIR,
          );
          logger.info(
-            `Re init: Cleaning up upload documents path ${uploadDocsPath}`,
+            `Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath}`,
          );
          try {
             await fs.promises.rm(uploadDocsPath, {
@@ -1025,11 +1026,10 @@ export class ProjectStore {
                }
             }
          } catch (error) {
+            const errorData = this.extractErrorDataFromError(error);
             logger.error(
                `Failed to download or mount location "${groupedLocation}"`,
-               {
-                  error,
-               },
+               errorData,
             );
             throw new PackageNotFoundError(
                `Failed to download or mount location: ${groupedLocation}`,
@@ -1075,9 +1075,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to download GCS directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download GCS directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download GCS directory: ${location}`,
             );
@@ -1093,9 +1095,11 @@ export class ProjectStore {
             await this.downloadGitHubDirectory(location, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to clone GitHub repository "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to clone GitHub repository "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to clone GitHub repository: ${location}`,
             );
@@ -1111,9 +1115,11 @@ export class ProjectStore {
             await this.downloadS3Directory(location, projectName, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to download S3 directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download S3 directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download S3 directory: ${location}`,
             );
@@ -1137,9 +1143,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to mount local directory "${packagePath}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to mount local directory "${packagePath}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to mount local directory: ${packagePath}`,
             );
@@ -1332,10 +1340,11 @@ export class ProjectStore {
       await new Promise<void>((resolve, reject) => {
          simpleGit().clone(repoUrl, absoluteDirPath, {}, (err) => {
             if (err) {
-               console.error(err);
-               logger.error(`Failed to clone GitHub repository "${repoUrl}"`, {
-                  error: err,
-               });
+               const errorData = this.extractErrorDataFromError(err);
+               logger.error(
+                  `Failed to clone GitHub repository "${repoUrl}"`,
+                  errorData,
+               );
                reject(err);
             }
             resolve();
@@ -1396,4 +1405,23 @@ export class ProjectStore {
 
       // https://github.com/credibledata/malloy-samples/imdb/publisher.json -> ${absoluteDirPath}/publisher.json
    }
+
+   private extractErrorDataFromError(error: unknown): {
+      error: string;
+      stack?: string;
+      task?: unknown;
+   } {
+      const errorMessage =
+         error instanceof Error ? error.message : String(error);
+      const errorData: { error: string; stack?: string; task?: unknown } = {
+         error: errorMessage,
+      };
+      if (error instanceof Error && logger.level === "debug") {
+         errorData.stack = error.stack;
+      }
+      if (error && typeof error === "object" && "task" in error) {
+         errorData.task = (error as { task?: unknown }).task;
+      }
+      return errorData;
+   }
 }

package/src/storage/StorageManager.ts

@@ -1,3 +1,4 @@
+import { logger } from "../logger";
 import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
@@ -37,11 +38,11 @@ export class StorageManager {
 
    async initialize(reinit: boolean = false): Promise<void> {
       if (reinit) {
-         console.log(
-            "RE-INITIALIZATION MODE: Database will be dropped and recreated",
+         logger.info(
+            "Reinitialization mode: Database will be dropped and recreated",
          );
       } else {
-         console.log("NORMAL MODE: Loading from existing database");
+         logger.info("Normal mode: Loading from existing database");
       }
 
       switch (this.config.type) {

package/src/storage/duckdb/schema.ts

@@ -1,3 +1,4 @@
+import { logger } from "../../logger";
 import { DuckDBConnection } from "./DuckDBConnection";
 
 export async function initializeSchema(
@@ -11,12 +12,12 @@ export async function initializeSchema(
    }
 
    if (force) {
-      console.log(
+      logger.info(
          "Reinitializing database schema dropping and recreating all tables",
       );
       await dropAllTables(db);
    } else {
-      console.log("Creating database schema for the first time...");
+      logger.info("Creating database schema for the first time...");
    }
 
    // Projects table
@@ -75,14 +76,14 @@ export async function initializeSchema(
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = ["packages", "connections", "projects"];
 
-   console.log("Dropping tables:", tables.join(", "));
+   logger.info("Dropping tables:", tables.join(", "));
 
    for (const table of tables) {
       try {
          await db.run(`DROP TABLE IF EXISTS ${table} `);
-         console.log(`Dropped table: ${table}`);
+         logger.info(`Dropped table: ${table}`);
       } catch (err) {
-         console.warn(` Warning: Could not drop table ${table}:`, err);
+         logger.warn(` Warning: Could not drop table ${table}:`, err);
       }
    }
 }

package/tests/harness/e2e.ts

@@ -7,6 +7,7 @@ import type {
 } from "@modelcontextprotocol/sdk/types.js";
 import http from "http";
 import path from "path";
+import { fileURLToPath } from "url";
 import { URL } from "url";
 
 /**
@@ -30,6 +31,9 @@ export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
    // 1.  Set SERVER_ROOT so ProjectStore loader finds publisher.config.json
    //--------------------------------------------------------------------------
    originalServerRoot = process.env.SERVER_ROOT;
+   // Use import.meta.url for cross-platform compatibility (works on Windows)
+   const __filename = fileURLToPath(import.meta.url);
+   const __dirname = path.dirname(__filename);
    const serverPackageDir = path.resolve(__dirname, "../../../"); // packages/server
    process.env.SERVER_ROOT = serverPackageDir;