@malloy-publisher/server 0.0.165 → 0.0.167

package/src/service/project_compile.spec.ts

@@ -0,0 +1,197 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs";
+import path from "path";
+import { extractPreamble, extractPreambleFromSource } from "./project";
+
+describe("extractPreambleFromSource", () => {
+   it("should extract pragmas and imports before a source definition", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import "utils.malloy"',
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+         '  dimension: name is "test"',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import "utils.malloy"',
+            'import { revenue } from "metrics.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should extract pragmas and imports before a run statement", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "run: my_source -> { aggregate: count() }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should extract pragmas and imports before a query definition", () => {
+      const content = [
+         'import { my_source } from "model.malloy"',
+         "",
+         "query: top_items is my_source -> { limit: 10 }",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import { my_source } from "model.malloy"');
+   });
+
+   it("should return empty string when file starts with source:", () => {
+      const content = 'source: my_source is duckdb.table("data.parquet")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with run:", () => {
+      const content = 'run: duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should return empty string when file starts with query:", () => {
+      const content = 'query: q is duckdb.sql("SELECT 1")';
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe("");
+   });
+
+   it("should preserve comments in the preamble", () => {
+      const content = [
+         "##! experimental.parameters",
+         "// This model defines revenue metrics",
+         'import { revenue } from "metrics.malloy"',
+         "// Main source below",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            "// This model defines revenue metrics",
+            'import { revenue } from "metrics.malloy"',
+            "// Main source below",
+         ].join("\n"),
+      );
+   });
+
+   it("should handle multiple import blocks", () => {
+      const content = [
+         'import { a } from "file_a.malloy"',
+         'import { b } from "file_b.malloy"',
+         'import { c, d } from "file_c.malloy"',
+         "",
+         "source: combined is a {",
+         "  join_one: b on b.id = a.b_id",
+         "}",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            'import { a } from "file_a.malloy"',
+            'import { b } from "file_b.malloy"',
+            'import { c, d } from "file_c.malloy"',
+         ].join("\n"),
+      );
+   });
+
+   it("should return all content when there are no source/query/run definitions", () => {
+      const content = [
+         "##! experimental.parameters",
+         'import { revenue } from "metrics.malloy"',
+         "",
+         "// no definitions yet",
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "##! experimental.parameters",
+            'import { revenue } from "metrics.malloy"',
+            "",
+            "// no definitions yet",
+         ].join("\n"),
+      );
+   });
+
+   it("should return empty string for empty input", () => {
+      expect(extractPreambleFromSource("")).toBe("");
+   });
+
+   it("should handle indented source/query/run definitions", () => {
+      const content = [
+         'import "model.malloy"',
+         "",
+         '  source: indented is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      // Indented definitions should still be detected (trimmed before check)
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe('import "model.malloy"');
+   });
+
+   it("should not stop on 'source' appearing in comments or strings", () => {
+      const content = [
+         "// This file defines the source of truth",
+         'import "model.malloy"',
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+
+      const result = extractPreambleFromSource(content);
+      expect(result).toBe(
+         [
+            "// This file defines the source of truth",
+            'import "model.malloy"',
+         ].join("\n"),
+      );
+   });
+});
+
+describe("extractPreamble (file-based)", () => {
+   const testDir = path.join(process.cwd(), "test-temp-preamble");
+   const testModelPath = path.join(testDir, "test_model.malloy");
+
+   beforeEach(() => {
+      if (!fs.existsSync(testDir)) {
+         fs.mkdirSync(testDir, { recursive: true });
+      }
+   });
+
+   afterEach(() => {
+      if (fs.existsSync(testDir)) {
+         fs.rmSync(testDir, { recursive: true, force: true });
+      }
+   });
+
+   it("should read and extract preamble from a file", async () => {
+      const content = [
+         'import { revenue } from "metrics.malloy"',
+         "",
+         'source: my_source is duckdb.table("data.parquet")',
+      ].join("\n");
+      fs.writeFileSync(testModelPath, content);
+
+      const result = await extractPreamble(testModelPath);
+      expect(result).toBe('import { revenue } from "metrics.malloy"');
+   });
+
+   it("should return empty string when file does not exist", async () => {
+      const result = await extractPreamble(
+         path.join(testDir, "nonexistent.malloy"),
+      );
+      expect(result).toBe("");
+   });
+});

package/src/service/project_store.ts

@@ -196,7 +196,8 @@ export class ProjectStore {
          );
       } catch (error) {
          markNotReady();
-         logger.error("Error initializing project store", { error });
+         const errorData = this.extractErrorDataFromError(error);
+         logger.error("Error initializing project store", errorData);
          process.exit(1);
       }
    }
@@ -533,7 +534,7 @@ export class ProjectStore {
             PUBLISHER_DATA_DIR,
          );
          logger.info(
-            `Re init: Cleaning up upload documents path ${uploadDocsPath}`,
+            `Reinitialization mode: Cleaning up upload documents path ${uploadDocsPath}`,
          );
          try {
             await fs.promises.rm(uploadDocsPath, {
@@ -1025,11 +1026,10 @@ export class ProjectStore {
                }
             }
          } catch (error) {
+            const errorData = this.extractErrorDataFromError(error);
             logger.error(
                `Failed to download or mount location "${groupedLocation}"`,
-               {
-                  error,
-               },
+               errorData,
             );
             throw new PackageNotFoundError(
                `Failed to download or mount location: ${groupedLocation}`,
@@ -1075,9 +1075,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to download GCS directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download GCS directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download GCS directory: ${location}`,
             );
@@ -1093,9 +1095,11 @@ export class ProjectStore {
             await this.downloadGitHubDirectory(location, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to clone GitHub repository "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to clone GitHub repository "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to clone GitHub repository: ${location}`,
             );
@@ -1111,9 +1115,11 @@ export class ProjectStore {
             await this.downloadS3Directory(location, projectName, targetPath);
             return;
          } catch (error) {
-            logger.error(`Failed to download S3 directory "${location}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to download S3 directory "${location}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to download S3 directory: ${location}`,
             );
@@ -1137,9 +1143,11 @@ export class ProjectStore {
             );
             return;
          } catch (error) {
-            logger.error(`Failed to mount local directory "${packagePath}"`, {
-               error,
-            });
+            const errorData = this.extractErrorDataFromError(error);
+            logger.error(
+               `Failed to mount local directory "${packagePath}"`,
+               errorData,
+            );
             throw new PackageNotFoundError(
                `Failed to mount local directory: ${packagePath}`,
             );
@@ -1332,10 +1340,11 @@ export class ProjectStore {
       await new Promise<void>((resolve, reject) => {
          simpleGit().clone(repoUrl, absoluteDirPath, {}, (err) => {
             if (err) {
-               console.error(err);
-               logger.error(`Failed to clone GitHub repository "${repoUrl}"`, {
-                  error: err,
-               });
+               const errorData = this.extractErrorDataFromError(err);
+               logger.error(
+                  `Failed to clone GitHub repository "${repoUrl}"`,
+                  errorData,
+               );
                reject(err);
             }
             resolve();
@@ -1396,4 +1405,23 @@ export class ProjectStore {
 
       // https://github.com/credibledata/malloy-samples/imdb/publisher.json -> ${absoluteDirPath}/publisher.json
    }
+
+   private extractErrorDataFromError(error: unknown): {
+      error: string;
+      stack?: string;
+      task?: unknown;
+   } {
+      const errorMessage =
+         error instanceof Error ? error.message : String(error);
+      const errorData: { error: string; stack?: string; task?: unknown } = {
+         error: errorMessage,
+      };
+      if (error instanceof Error && logger.level === "debug") {
+         errorData.stack = error.stack;
+      }
+      if (error && typeof error === "object" && "task" in error) {
+         errorData.task = (error as { task?: unknown }).task;
+      }
+      return errorData;
+   }
 }

package/src/storage/StorageManager.ts

@@ -1,3 +1,4 @@
+import { logger } from "../logger";
 import { DatabaseConnection, ResourceRepository } from "./DatabaseInterface";
 import { DuckDBConnection } from "./duckdb/DuckDBConnection";
 import { DuckDBRepository } from "./duckdb/DuckDBRepository";
@@ -37,11 +38,11 @@ export class StorageManager {
 
    async initialize(reinit: boolean = false): Promise<void> {
       if (reinit) {
-         console.log(
-            "RE-INITIALIZATION MODE: Database will be dropped and recreated",
+         logger.info(
+            "Reinitialization mode: Database will be dropped and recreated",
          );
       } else {
-         console.log("NORMAL MODE: Loading from existing database");
+         logger.info("Normal mode: Loading from existing database");
       }
 
       switch (this.config.type) {

package/src/storage/duckdb/schema.ts

@@ -1,3 +1,4 @@
+import { logger } from "../../logger";
 import { DuckDBConnection } from "./DuckDBConnection";
 
 export async function initializeSchema(
@@ -11,12 +12,12 @@ export async function initializeSchema(
    }
 
    if (force) {
-      console.log(
+      logger.info(
          "Reinitializing database schema dropping and recreating all tables",
       );
       await dropAllTables(db);
    } else {
-      console.log("Creating database schema for the first time...");
+      logger.info("Creating database schema for the first time...");
    }
 
    // Projects table
@@ -75,14 +76,14 @@ export async function initializeSchema(
 async function dropAllTables(db: DuckDBConnection): Promise<void> {
    const tables = ["packages", "connections", "projects"];
 
-   console.log("Dropping tables:", tables.join(", "));
+   logger.info("Dropping tables:", tables.join(", "));
 
    for (const table of tables) {
       try {
          await db.run(`DROP TABLE IF EXISTS ${table} `);
-         console.log(`Dropped table: ${table}`);
+         logger.info(`Dropped table: ${table}`);
       } catch (err) {
-         console.warn(` Warning: Could not drop table ${table}:`, err);
+         logger.warn(` Warning: Could not drop table ${table}:`, err);
       }
    }
 }

package/tests/harness/e2e.ts

@@ -7,6 +7,7 @@ import type {
 } from "@modelcontextprotocol/sdk/types.js";
 import http from "http";
 import path from "path";
+import { fileURLToPath } from "url";
 import { URL } from "url";
 
 /**
@@ -30,6 +31,9 @@ export async function startE2E(): Promise<E2EEnv & { stop(): Promise<void> }> {
    // 1.  Set SERVER_ROOT so ProjectStore loader finds publisher.config.json
    //--------------------------------------------------------------------------
    originalServerRoot = process.env.SERVER_ROOT;
+   // Use import.meta.url for cross-platform compatibility (works on Windows)
+   const __filename = fileURLToPath(import.meta.url);
+   const __dirname = path.dirname(__filename);
    const serverPackageDir = path.resolve(__dirname, "../../../"); // packages/server
    process.env.SERVER_ROOT = serverPackageDir;
 

package/tests/harness/mcp_test_setup.ts

@@ -8,6 +8,7 @@ import {
 import http from "http";
 import { AddressInfo } from "net";
 import path from "path";
+import { fileURLToPath } from "url";
 import { URL } from "url";
 
 // --- Real Server Import ---
@@ -31,8 +32,11 @@ export async function setupE2ETestEnvironment(): Promise<McpE2ETestEnvironment>
    // --- Store and Set SERVER_ROOT Env Var ---
    // The ProjectStore relies on SERVER_ROOT to find publisher.config.json.
    originalServerRoot = process.env.SERVER_ROOT; // Store original value
-   // Resolve the path to 'packages/server' based on the location of this file (__dirname)
-   const serverPackageDir = path.resolve(__dirname, "../../"); // Go up two levels from .../packages/server/src/__tests__
+   // Resolve the path to 'packages/server' based on the location of this file
+   // Use import.meta.url for cross-platform compatibility (works on Windows)
+   const __filename = fileURLToPath(import.meta.url);
+   const __dirname = path.dirname(__filename);
+   const serverPackageDir = path.resolve(__dirname, "../../"); // Go up two levels from .../packages/server/tests/harness
    process.env.SERVER_ROOT = serverPackageDir;
    console.log(
       `[E2E Test Setup] Temporarily set SERVER_ROOT=${process.env.SERVER_ROOT}`,