@malloy-publisher/server 0.0.162 → 0.0.167

package/src/service/project.ts

@@ -1,3 +1,5 @@
+import type { LogMessage } from "@malloydata/malloy";
+import { FixedConnectionMap, MalloyError, Runtime } from "@malloydata/malloy";
 import { BaseConnection } from "@malloydata/malloy/connection";
 import { Mutex } from "async-mutex";
 import * as fs from "fs";
@@ -10,6 +12,7 @@ import {
    ProjectNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
+import { URL_READER } from "../utils";
 import { createProjectConnections, InternalConnection } from "./connection";
 import { ApiConnection } from "./model";
 import { Package } from "./package";
@@ -159,6 +162,71 @@ export class Project {
       return this.metadata;
    }
 
+   public async compileSource(
+      packageName: string,
+      modelName: string,
+      source: string,
+      includeSql: boolean = false,
+   ): Promise<{ problems: LogMessage[]; sql?: string }> {
+      // Place the virtual file in the model's directory so relative imports resolve correctly.
+      const modelDir = path.dirname(
+         path.join(this.projectPath, packageName, modelName),
+      );
+      const virtualUri = `file://${path.join(modelDir, "__compile_check.malloy")}`;
+      const virtualUrl = new URL(virtualUri);
+
+      // Read the model file and extract its preamble (pragmas + imports) so that
+      // the user's query inherits the model's import context.
+      const modelPath = path.join(this.projectPath, packageName, modelName);
+      const preamble = await extractPreamble(modelPath);
+      const fullSource = preamble ? `${preamble}\n${source}` : source;
+
+      // Create a URL Reader that serves the source string for the virtual file,
+      // but falls back to the disk for everything else (imports).
+      const interceptingReader = {
+         readURL: async (url: URL) => {
+            if (url.toString() === virtualUri) {
+               return fullSource;
+            }
+            return URL_READER.readURL(url);
+         },
+      };
+
+      // Initialize Runtime with the project's active connections
+      const runtime = new Runtime({
+         urlReader: interceptingReader,
+         connections: new FixedConnectionMap(this.malloyConnections, "duckdb"),
+      });
+
+      // Attempt to compile
+      try {
+         const modelMaterializer = runtime.loadModel(virtualUrl);
+         const model = await modelMaterializer.getModel();
+
+         // If includeSql is requested and compilation succeeded, attempt to extract SQL
+         let sql: string | undefined;
+         if (includeSql) {
+            try {
+               const queryMaterializer = modelMaterializer.loadFinalQuery();
+               sql = await queryMaterializer.getSQL();
+            } catch {
+               // Source may not contain a runnable query (e.g. only source definitions),
+               // in which case we simply omit the sql field.
+            }
+         }
+
+         // If successful, return any non-fatal warnings
+         return { problems: model.problems, sql };
+      } catch (error) {
+         // If parsing/compilation fails, return the errors
+         if (error instanceof MalloyError) {
+            return { problems: error.problems };
+         }
+         // If it's a system error (e.g. file not found), throw it up
+         throw error;
+      }
+   }
+
    public listApiConnections(): ApiConnection[] {
       return this.apiConnections;
    }
@@ -244,11 +312,18 @@ export class Project {
       // package multiple times.
       let packageMutex = this.packageMutexes.get(packageName);
       if (packageMutex?.isLocked()) {
+         logger.debug(
+            `Package ${packageName} is being loaded, waiting for unlock...`,
+         );
          await packageMutex.waitForUnlock();
+         logger.debug(`Package ${packageName} unlocked`);
          const existingPackage = this.packages.get(packageName);
          if (existingPackage) {
+            logger.debug(`Package ${packageName} loaded by another request`);
             return existingPackage;
          }
+         // If package still doesn't exist after unlock, it might have failed to load
+         // Continue to try loading it ourselves
       }
       packageMutex = new Mutex();
       this.packageMutexes.set(packageName, packageMutex);
@@ -264,19 +339,23 @@ export class Project {
          this.setPackageStatus(packageName, PackageStatus.LOADING);
 
          try {
+            logger.debug(`Loading package ${packageName}...`);
+            const packagePath = path.join(this.projectPath, packageName);
             const _package = await Package.create(
                this.projectName,
                packageName,
-               path.join(this.projectPath, packageName),
+               packagePath,
                this.malloyConnections,
             );
             this.packages.set(packageName, _package);
 
             // Set package status to serving
             this.setPackageStatus(packageName, PackageStatus.SERVING);
+            logger.debug(`Successfully loaded package ${packageName}`);
 
             return _package;
          } catch (error) {
+            logger.error(`Failed to load package ${packageName}`, { error });
             // Clean up on error - mutex will be automatically released by runExclusive
             this.packages.delete(packageName);
             this.packageStatuses.delete(packageName);
@@ -537,3 +616,43 @@ export class Project {
          });
    }
 }
+
+/**
+ * Extracts the preamble from a Malloy model file — the leading block of
+ * `##!` pragmas, `import` statements, blank lines, and comments that appear
+ * before any `source:`, `query:`, or `run:` definition. This allows a
+ * submitted query to inherit the model's import context.
+ */
+export async function extractPreamble(modelPath: string): Promise<string> {
+   try {
+      const content = await fs.promises.readFile(modelPath, "utf8");
+      return extractPreambleFromSource(content);
+   } catch {
+      // If the model file can't be read, return empty preamble
+      // and let the compilation surface any import errors naturally.
+      return "";
+   }
+}
+
+/**
+ * Extracts the preamble from Malloy source text. Exported for testing.
+ */
+export function extractPreambleFromSource(content: string): string {
+   const lines = content.split("\n");
+   const preambleLines: string[] = [];
+
+   for (const line of lines) {
+      const trimmed = line.trim();
+      // Stop at the first source/query/run definition
+      if (
+         trimmed.startsWith("source:") ||
+         trimmed.startsWith("query:") ||
+         trimmed.startsWith("run:")
+      ) {
+         break;
+      }
+      preambleLines.push(line);
+   }
+
+   return preambleLines.join("\n").trimEnd();
+}

package/src/service/project_compile.spec.ts

@@ -0,0 +1,197 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs";
+import path from "path";
+import { extractPreamble, extractPreambleFromSource } from "./project";
+
+describe("extractPreambleFromSource", () => {
+   it("should extract pragmas and imports before a source definition", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import "utils.malloy"',
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+         '  dimension: name is "test"',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import "utils.malloy"',
+            'import { revenue } from "metrics.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should extract pragmas and imports before a run statement", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "run: my_source -> { aggregate: count() }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should extract pragmas and imports before a query definition", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "query: top_items is my_source -> { limit: 10 }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should return empty string when file starts with source:", () => {
+      const content = 'source: my_source is duckdb.table("data.parquet")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with run:", () => {
+      const content = 'run: duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with query:", () => {
+      const content = 'query: q is duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should preserve comments in the preamble", () => {
+      const content = [
+         "##! experimental.parameters",
+         "// This model defines revenue metrics",
+         'import { revenue } from "metrics.malloy"',
+         "// Main source below",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            "// This model defines revenue metrics",
+            'import { revenue } from "metrics.malloy"',
+            "// Main source below",
+         ].join("\n"),
+      );
+   });
+
+   it("should handle multiple import blocks", () => {
+      const content = [
+         'import { a } from "file_a.malloy"',
+         'import { b } from "file_b.malloy"',
+         'import { c, d } from "file_c.malloy"',
+         "",
+         "source: combined is a {",
+         "  join_one: b on b.id = a.b_id",
+         "}",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            'import { a } from "file_a.malloy"',
+            'import { b } from "file_b.malloy"',
+            'import { c, d } from "file_c.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should return all content when there are no source/query/run definitions", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import { revenue } from "metrics.malloy"',
+         "",
+         "// no definitions yet",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import { revenue } from "metrics.malloy"',
+            "",
+            "// no definitions yet",
+         ].join("\n"),
+      );
+   });
+
+   it("should return empty string for empty input", () => {
+      expect(extractPreambleFromSource("")).toBe("");
+   });
+
+   it("should handle indented source/query/run definitions", () => {
+      const content = [
+         'import "model.malloy"',
+         "",
+         '  source: indented is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      // Indented definitions should still be detected (trimmed before check)
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import "model.malloy"');
+   });
+
+   it("should not stop on 'source' appearing in comments or strings", () => {
+      const content = [
+         "// This file defines the source of truth",
+         'import "model.malloy"',
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "// This file defines the source of truth",
+            'import "model.malloy"',
+         ].join("\n"),
+      );
+   });
+});
+
+describe("extractPreamble (file-based)", () => {
+   const testDir = path.join(process.cwd(), "test-temp-preamble");
+   const testModelPath = path.join(testDir, "test_model.malloy");
+
+   beforeEach(() => {
+      if (!fs.existsSync(testDir)) {
+         fs.mkdirSync(testDir, { recursive: true });
+      }
+   });
+
+   afterEach(() => {
+      if (fs.existsSync(testDir)) {
+         fs.rmSync(testDir, { recursive: true, force: true });
+      }
+   });
+
+   it("should read and extract preamble from a file", async () => {
+      const content = [
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+      fs.writeFileSync(testModelPath, content);
+
+      const result = await extractPreamble(testModelPath);
+      expect(result).toBe('import { revenue } from "metrics.malloy"');
+   });
+
+   it("should return empty string when file does not exist", async () => {
+      const result = await extractPreamble(
+         path.join(testDir, "nonexistent.malloy"),
+      );
+      expect(result).toBe("");
+   });
+});

package/src/service/project_store.ts

@@ -24,6 +24,7 @@ import {
    PackageNotFoundError,
    ProjectNotFoundError,
 } from "../errors";
+import { getOperationalState, markNotReady, markReady } from "../health";
 import { formatDuration, logger } from "../logger";
 import { Connection } from "../storage/DatabaseInterface";
 import { StorageConfig, StorageManager } from "../storage/StorageManager";
@@ -188,12 +189,15 @@ export class ProjectStore {
          }
 
          this.isInitialized = true;
+         markReady();
          const initializationDuration = performance.now() - initialTime;
          logger.info(
             `Project store successfully initialized in ${formatDuration(initializationDuration)}`,
          );
       } catch (error) {
-         logger.error("Error initializing project store", { error });
+         markNotReady();
+         const errorData = this.extractErrorDataFromError(error);
+         logger.error("Error initializing project store", errorData);
          process.exit(1);
       }
    }
@@ -530,7 +534,7 @@ export class ProjectStore {
             PUBLISHER_DATA_DIR,
          );
          logger.info(
-            `Re init: Cleaning up upload documents path ${uploadDocsPath}`,
+            `Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath}`,
          );
          try {
             await fs.promises.rm(uploadDocsPath, {
@@ -572,6 +576,8 @@ export class ProjectStore {
          projects: [] as Array<components["schemas"]["Project"]>,
          initialized: this.isInitialized,
          frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
+         operationalState:
+            getOperationalState() as components["schemas"]["ServerStatus"]["operationalState"],
       };
 
       const projects = await this.listProjects(true);
@@ -1020,11 +1026,10 @@ export class ProjectStore {
                }
             }
          } catch (error) {
+            const errorData = this.extractErrorDataFromError(error);
             logger.error(
                `Failed to download or mount location "${groupedLocation}"`,
-               {
-                  error,
-               },
+               errorData,
             );
             throw new PackageNotFoundError(
                `Failed to download or mount location: ${groupedLocation}`,
@@ -1070,9 +1075,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to download GCS directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download GCS directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download GCS directory: ${location}`,
             );
@@ -1088,9 +1095,11 @@ export class ProjectStore {
             await this.downloadGitHubDirectory(location, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to clone GitHub repository "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to clone GitHub repository "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to clone GitHub repository: ${location}`,
             );
@@ -1106,9 +1115,11 @@ export class ProjectStore {
             await this.downloadS3Directory(location, projectName, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to download S3 directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download S3 directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download S3 directory: ${location}`,
             );
@@ -1132,9 +1143,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to mount local directory "${packagePath}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to mount local directory "${packagePath}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to mount local directory: ${packagePath}`,
             );
@@ -1327,10 +1340,11 @@ export class ProjectStore {
       await new Promise<void>((resolve, reject) => {
          simpleGit().clone(repoUrl, absoluteDirPath, {}, (err) => {
             if (err) {
-               console.error(err);
-               logger.error(`Failed to clone GitHub repository "${repoUrl}"`, {
-                  error: err,
-               });
+               const errorData = this.extractErrorDataFromError(err);
+               logger.error(
+                  `Failed to clone GitHub repository "${repoUrl}"`,
+                  errorData,
+               );
                reject(err);
             }
             resolve();
@@ -1391,4 +1405,23 @@ export class ProjectStore {
 
       // https://github.com/credibledata/malloy-samples/imdb/publisher.json -> ${absoluteDirPath}/publisher.json
    }
+
+   private extractErrorDataFromError(error: unknown): {
+      error: string;
+      stack?: string;
+      task?: unknown;
+   } {
+      const errorMessage =
+         error instanceof Error ? error.message : String(error);
+      const errorData: { error: string; stack?: string; task?: unknown } = {
+         error: errorMessage,
+      };
+      if (error instanceof Error && logger.level === "debug") {
+         errorData.stack = error.stack;
+      }
+      if (error && typeof error === "object" && "task" in error) {
+         errorData.task = (error as { task?: unknown }).task;
+      }
+      return errorData;
+   }
 }

package/src/storage/StorageManager.ts

@@ -1,3 +1,4 @@
+import { logger } from "../logger";
 import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
@@ -37,11 +38,11 @@ export class StorageManager {
 
    async initialize(reinit: boolean = false): Promise<void> {
       if (reinit) {
-         console.log(
-            "RE-INITIALIZATION MODE: Database will be dropped and recreated",
+         logger.info(
+            "Reinitialization mode: Database will be dropped and recreated",
          );
       } else {
-         console.log("NORMAL MODE: Loading from existing database");
+         logger.info("Normal mode: Loading from existing database");
       }
 
       switch (this.config.type) {

package/src/storage/duckdb/schema.ts

@@ -1,3 +1,4 @@
+import { logger } from "../../logger";
 import { DuckDBConnection } from "./DuckDBConnection";
 
 export async function initializeSchema(
@@ -11,12 +12,12 @@ export async function initializeSchema(
    }
 
    if (force) {
-      console.log(
+      logger.info(
          "Reinitializing database schema dropping and recreating all tables",
       );
       await dropAllTables(db);
    } else {
-      console.log("Creating database schema for the first time...");
+      logger.info("Creating database schema for the first time...");
    }
 
    // Projects table
@@ -75,14 +76,14 @@ export async function initializeSchema(
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = ["packages", "connections", "projects"];
 
-   console.log("Dropping tables:", tables.join(", "));
+   logger.info("Dropping tables:", tables.join(", "));
 
    for (const table of tables) {
       try {
          await db.run(`DROP TABLE IF EXISTS ${table} `);
-         console.log(`Dropped table: ${table}`);
+         logger.info(`Dropped table: ${table}`);
       } catch (err) {
-         console.warn(` Warning: Could not drop table ${table}:`, err);
+         logger.warn(` Warning: Could not drop table ${table}:`, err);
       }
    }
 }

package/tests/harness/e2e.ts

@@ -7,6 +7,7 @@ import type {
 } from "@modelcontextprotocol/sdk/types.js";
 import http from "http";
 import path from "path";
+import { fileURLToPath } from "url";
 import { URL } from "url";
 
 /**
@@ -30,6 +31,9 @@ export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
    // 1.  Set SERVER_ROOT so ProjectStore loader finds publisher.config.json
    //--------------------------------------------------------------------------
    originalServerRoot = process.env.SERVER_ROOT;
+   // Use import.meta.url for cross-platform compatibility (works on Windows)
+   const __filename = fileURLToPath(import.meta.url);
+   const __dirname = path.dirname(__filename);
    const serverPackageDir = path.resolve(__dirname, "../../../"); // packages/server
    process.env.SERVER_ROOT = serverPackageDir;
 

package/tests/harness/mcp_test_setup.ts

@@ -8,6 +8,7 @@ import {
 import http from "http";
 import { AddressInfo } from "net";
 import path from "path";
+import { fileURLToPath } from "url";
 import { URL } from "url";
 
 // --- Real Server Import ---
@@ -31,8 +32,11 @@ export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment>
    // --- Store and Set SERVER_ROOT Env Var ---
    // The ProjectStore relies on SERVER_ROOT to find publisher.config.json.
    originalServerRoot = process.env.SERVER_ROOT; // Store original value
-   // Resolve the path to 'packages/server' based on the location of this file (__dirname)
-   const serverPackageDir = path.resolve(__dirname, "../../"); // Go up two levels from .../packages/server/src/__tests__
+   // Resolve the path to 'packages/server' based on the location of this file
+   // Use import.meta.url for cross-platform compatibility (works on Windows)
+   const __filename = fileURLToPath(import.meta.url);
+   const __dirname = path.dirname(__filename);
+   const serverPackageDir = path.resolve(__dirname, "../../"); // Go up two levels from .../packages/server/tests/harness
    process.env.SERVER_ROOT = serverPackageDir;
    console.log(
       `[E2E Test Setup] Temporarily set SERVER_ROOT=${process.env.SERVER_ROOT}`,

package/dist/app/assets/HomePage-CgSTNEQW.js

@@ -1 +0,0 @@
-import{t as o,j as a,Q as e}from"./index-BoduKqUa.js";function n(){const t=o();return a.jsx(e,{onClickProject:t})}export{n as default};

package/dist/app/assets/PackagePage-BHvPFsmZ.js

@@ -1 +0,0 @@
-import{p as n,t,j as e,A as c,J as o}from"./index-BoduKqUa.js";function l(){const{projectName:s,packageName:a}=n(),r=t();if(s)if(a){const i=c({projectName:s,packageName:a});return e.jsx(o,{onClickPackageFile:r,resourceUri:i})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing package name"})});else return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})})}export{l as default};

package/dist/app/assets/ProjectPage-ndnv5nGp.js

@@ -1 +0,0 @@
-import{t as a,p as n,j as e,A as c,E as o}from"./index-BoduKqUa.js";function j(){const t=a(),{projectName:s}=n();if(s){const r=c({projectName:s});return e.jsx(o,{onSelectPackage:t,resourceUri:r})}else return e.jsx("div",{children:e.jsx("h2",{children:"Missing project name"})})}export{j as default};

package/dist/app/assets/RouteError-CZC9OOKh.js

@@ -1 +0,0 @@
-import{F as o,j as r,z as s,S as n,q as t,T as a}from"./index-BoduKqUa.js";function x(){const e=o();return console.error(e),r.jsx(s,{maxWidth:"lg",component:"main",sx:{display:"flex",flexDirection:"column",my:2,gap:0},children:r.jsxs(n,{sx:{m:"auto",flexDirection:"column"},children:[r.jsx(t,{sx:{height:"300px"}}),r.jsx("img",{src:"/error.png"}),r.jsx(a,{variant:"subtitle1",children:"An unexpected error occurred"})]})})}export{x as default};