@malloy-publisher/server 0.0.83 → 0.0.85

package/dist/server.js

@@ -235636,8 +235636,8 @@ class DatabaseController {
     this.projectStore = projectStore;
   }
   async listDatabases(projectName, packageName) {
-    const project = await this.projectStore.getProject(projectName);
-    const p = await project.getPackage(packageName);
+    const project = await this.projectStore.getProject(projectName, false);
+    const p = await project.getPackage(packageName, false);
     return p.listDatabases();
   }
 }
@@ -243482,6 +243482,214 @@ function initializeMcpServer(projectStore) {
 var fs7 = __toESM(require("fs/promises"));
 var path6 = __toESM(require("path"));
 
+// ../../node_modules/async-mutex/index.mjs
+var E_TIMEOUT = new Error("timeout while waiting for mutex to become available");
+var E_ALREADY_LOCKED = new Error("mutex already locked");
+var E_CANCELED = new Error("request for lock canceled");
+var __awaiter$2 = function(thisArg, _arguments, P, generator) {
+  function adopt(value) {
+    return value instanceof P ? value : new P(function(resolve) {
+      resolve(value);
+    });
+  }
+  return new (P || (P = Promise))(function(resolve, reject) {
+    function fulfilled(value) {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function rejected(value) {
+      try {
+        step(generator["throw"](value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function step(result) {
+      result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
+    }
+    step((generator = generator.apply(thisArg, _arguments || [])).next());
+  });
+};
+
+class Semaphore {
+  constructor(_value, _cancelError = E_CANCELED) {
+    this._value = _value;
+    this._cancelError = _cancelError;
+    this._queue = [];
+    this._weightedWaiters = [];
+  }
+  acquire(weight = 1, priority = 0) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    return new Promise((resolve, reject) => {
+      const task = { resolve, reject, weight, priority };
+      const i = findIndexFromEnd(this._queue, (other) => priority <= other.priority);
+      if (i === -1 && weight <= this._value) {
+        this._dispatchItem(task);
+      } else {
+        this._queue.splice(i + 1, 0, task);
+      }
+    });
+  }
+  runExclusive(callback_1) {
+    return __awaiter$2(this, arguments, undefined, function* (callback, weight = 1, priority = 0) {
+      const [value, release] = yield this.acquire(weight, priority);
+      try {
+        return yield callback(value);
+      } finally {
+        release();
+      }
+    });
+  }
+  waitForUnlock(weight = 1, priority = 0) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    if (this._couldLockImmediately(weight, priority)) {
+      return Promise.resolve();
+    } else {
+      return new Promise((resolve) => {
+        if (!this._weightedWaiters[weight - 1])
+          this._weightedWaiters[weight - 1] = [];
+        insertSorted(this._weightedWaiters[weight - 1], { resolve, priority });
+      });
+    }
+  }
+  isLocked() {
+    return this._value <= 0;
+  }
+  getValue() {
+    return this._value;
+  }
+  setValue(value) {
+    this._value = value;
+    this._dispatchQueue();
+  }
+  release(weight = 1) {
+    if (weight <= 0)
+      throw new Error(`invalid weight ${weight}: must be positive`);
+    this._value += weight;
+    this._dispatchQueue();
+  }
+  cancel() {
+    this._queue.forEach((entry) => entry.reject(this._cancelError));
+    this._queue = [];
+  }
+  _dispatchQueue() {
+    this._drainUnlockWaiters();
+    while (this._queue.length > 0 && this._queue[0].weight <= this._value) {
+      this._dispatchItem(this._queue.shift());
+      this._drainUnlockWaiters();
+    }
+  }
+  _dispatchItem(item) {
+    const previousValue = this._value;
+    this._value -= item.weight;
+    item.resolve([previousValue, this._newReleaser(item.weight)]);
+  }
+  _newReleaser(weight) {
+    let called = false;
+    return () => {
+      if (called)
+        return;
+      called = true;
+      this.release(weight);
+    };
+  }
+  _drainUnlockWaiters() {
+    if (this._queue.length === 0) {
+      for (let weight = this._value;weight > 0; weight--) {
+        const waiters = this._weightedWaiters[weight - 1];
+        if (!waiters)
+          continue;
+        waiters.forEach((waiter) => waiter.resolve());
+        this._weightedWaiters[weight - 1] = [];
+      }
+    } else {
+      const queuedPriority = this._queue[0].priority;
+      for (let weight = this._value;weight > 0; weight--) {
+        const waiters = this._weightedWaiters[weight - 1];
+        if (!waiters)
+          continue;
+        const i = waiters.findIndex((waiter) => waiter.priority <= queuedPriority);
+        (i === -1 ? waiters : waiters.splice(0, i)).forEach((waiter) => waiter.resolve());
+      }
+    }
+  }
+  _couldLockImmediately(weight, priority) {
+    return (this._queue.length === 0 || this._queue[0].priority < priority) && weight <= this._value;
+  }
+}
+function insertSorted(a, v) {
+  const i = findIndexFromEnd(a, (other) => v.priority <= other.priority);
+  a.splice(i + 1, 0, v);
+}
+function findIndexFromEnd(a, predicate) {
+  for (let i = a.length - 1;i >= 0; i--) {
+    if (predicate(a[i])) {
+      return i;
+    }
+  }
+  return -1;
+}
+var __awaiter$1 = function(thisArg, _arguments, P, generator) {
+  function adopt(value) {
+    return value instanceof P ? value : new P(function(resolve) {
+      resolve(value);
+    });
+  }
+  return new (P || (P = Promise))(function(resolve, reject) {
+    function fulfilled(value) {
+      try {
+        step(generator.next(value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function rejected(value) {
+      try {
+        step(generator["throw"](value));
+      } catch (e) {
+        reject(e);
+      }
+    }
+    function step(result) {
+      result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected);
+    }
+    step((generator = generator.apply(thisArg, _arguments || [])).next());
+  });
+};
+
+class Mutex {
+  constructor(cancelError) {
+    this._semaphore = new Semaphore(1, cancelError);
+  }
+  acquire() {
+    return __awaiter$1(this, arguments, undefined, function* (priority = 0) {
+      const [, releaser] = yield this._semaphore.acquire(1, priority);
+      return releaser;
+    });
+  }
+  runExclusive(callback, priority = 0) {
+    return this._semaphore.runExclusive(() => callback(), 1, priority);
+  }
+  isLocked() {
+    return this._semaphore.isLocked();
+  }
+  waitForUnlock(priority = 0) {
+    return this._semaphore.waitForUnlock(1, priority);
+  }
+  release() {
+    if (this._semaphore.isLocked())
+      this._semaphore.release();
+  }
+  cancel() {
+    return this._semaphore.cancel();
+  }
+}
+
 // src/service/project.ts
 var fs6 = __toESM(require("fs/promises"));
 var path5 = __toESM(require("path"));
@@ -243559,7 +243767,7 @@ async function createConnections(basePath) {
           const config = {
             host: connection.mysqlConnection.host,
             port: connection.mysqlConnection.port,
-            username: connection.mysqlConnection.user,
+            user: connection.mysqlConnection.user,
             password: connection.mysqlConnection.password,
             database: connection.mysqlConnection.database
           };
@@ -244198,6 +244406,8 @@ class Scheduler {
 }
 
 // src/service/package.ts
+var ENABLE_LIST_MODEL_COMPILATION = true;
+
 class Package {
   projectName;
   packageName;
@@ -244223,18 +244433,57 @@ class Package {
   static async create(projectName, packageName, packagePath, projectConnections) {
     const startTime = performance.now();
     await Package.validatePackageManifestExistsOrThrowError(packagePath);
+    const manifestValidationTime = performance.now();
+    logger.info("Package manifest validation completed", {
+      packageName,
+      duration: manifestValidationTime - startTime,
+      unit: "ms"
+    });
     try {
       const packageConfig = await Package.readPackageConfig(packagePath);
+      const packageConfigTime = performance.now();
+      logger.info("Package config read completed", {
+        packageName,
+        duration: packageConfigTime - manifestValidationTime,
+        unit: "ms"
+      });
       packageConfig.resource = `${API_PREFIX}/projects/${projectName}/packages/${packageName}`;
       const databases = await Package.readDatabases(packagePath);
+      const databasesTime = performance.now();
+      logger.info("Databases read completed", {
+        packageName,
+        databaseCount: databases.length,
+        duration: databasesTime - packageConfigTime,
+        unit: "ms"
+      });
       const connections = new Map(projectConnections);
       const { malloyConnections: packageConnections } = await createConnections(packagePath);
+      const connectionsTime = performance.now();
+      logger.info("Package connections created", {
+        packageName,
+        connectionCount: packageConnections.size,
+        duration: connectionsTime - databasesTime,
+        unit: "ms"
+      });
       packageConnections.forEach((connection) => {
         connections.set(connection.name, connection);
       });
       connections.set("duckdb", new import_db_duckdb.DuckDBConnection("duckdb", ":memory:", packagePath));
       const models = await Package.loadModels(packageName, packagePath, connections);
+      const modelsTime = performance.now();
+      logger.info("Models loaded", {
+        packageName,
+        modelCount: models.size,
+        duration: modelsTime - connectionsTime,
+        unit: "ms"
+      });
       const scheduler = Scheduler.create(models);
+      const schedulerTime = performance.now();
+      logger.info("Scheduler created", {
+        packageName,
+        duration: schedulerTime - modelsTime,
+        unit: "ms"
+      });
       const endTime = performance.now();
       const executionTime = endTime - startTime;
       this.packageLoadHistogram.record(executionTime, {
@@ -244243,14 +244492,16 @@ class Package {
       });
       return new Package(projectName, packageName, packagePath, packageConfig, databases, models, scheduler);
     } catch (error) {
-      logger.error("Error loading package", { error });
+      logger.error(`Error loading package ${packageName}`, { error });
       const endTime = performance.now();
       const executionTime = endTime - startTime;
       this.packageLoadHistogram.record(executionTime, {
         malloy_package_name: packageName,
         status: "error"
       });
-      throw new Error("Error loading package: " + error);
+      throw new Error(`Error loading package ${packageName}`, {
+        cause: error
+      });
     }
   }
   getPackageName() {
@@ -244280,10 +244531,12 @@ class Package {
       return modelPath.endsWith(MODEL_FILE_SUFFIX);
     }).map(async (modelPath) => {
       let error;
-      try {
-        await this.models.get(modelPath)?.getModel();
-      } catch (modelError) {
-        error = modelError instanceof Error ? modelError.message : undefined;
+      if (ENABLE_LIST_MODEL_COMPILATION) {
+        try {
+          await this.models.get(modelPath)?.getModel();
+        } catch (modelError) {
+          error = modelError instanceof Error ? modelError.message : undefined;
+        }
       }
       return {
         projectName: this.projectName,
@@ -244298,7 +244551,10 @@ class Package {
     return await Promise.all(Array.from(this.models.keys()).filter((modelPath) => {
       return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
     }).map(async (modelPath) => {
-      const error = this.models.get(modelPath)?.getNotebookError();
+      let error;
+      if (ENABLE_LIST_MODEL_COMPILATION) {
+        error = this.models.get(modelPath)?.getNotebookError();
+      }
       return {
         projectName: this.projectName,
         packageName: this.packageName,
@@ -244380,6 +244636,7 @@ class Package {
 // src/service/project.ts
 class Project {
   packages = new Map;
+  packageMutexes = new Map;
   malloyConnections;
   apiConnections;
   internalConnections;
@@ -244448,10 +244705,9 @@ class Project {
       });
       const packageMetadata = await Promise.all(files.filter((file) => file.isDirectory()).map(async (directory) => {
         try {
-          const _package = await this.getPackage(directory.name, false);
-          const metadata = _package.getPackageMetadata();
-          return metadata;
-        } catch {
+          return (await this.getPackage(directory.name, false)).getPackageMetadata();
+        } catch (error) {
+          console.log(`Failed to load package: ${directory.name} due to : ${error}`);
           return;
         }
       }));
@@ -244462,17 +244718,25 @@ class Project {
     }
   }
   async getPackage(packageName, reload) {
-    let _package = this.packages.get(packageName);
-    if (_package === undefined || reload) {
+    let packageMutex = this.packageMutexes.get(packageName);
+    if (!packageMutex) {
+      packageMutex = new Mutex;
+      this.packageMutexes.set(packageName, packageMutex);
+    }
+    return await packageMutex.runExclusive(async () => {
+      const _package = this.packages.get(packageName);
+      if (_package !== undefined && !reload) {
+        return _package;
+      }
       try {
-        _package = await Package.create(this.projectName, packageName, path5.join(this.projectPath, packageName), this.malloyConnections);
-        this.packages.set(packageName, _package);
+        const _package2 = await Package.create(this.projectName, packageName, path5.join(this.projectPath, packageName), this.malloyConnections);
+        this.packages.set(packageName, _package2);
+        return _package2;
       } catch (error) {
         this.packages.delete(packageName);
         throw error;
       }
-    }
-    return _package;
+    });
   }
 }
 
@@ -244595,6 +244859,7 @@ var databaseController = new DatabaseController(projectStore);
 var queryController = new QueryController(projectStore);
 var scheduleController = new ScheduleController(projectStore);
 var mcpApp = import_express.default();
+initProjects();
 mcpApp.use(MCP_ENDPOINT, import_express.default.json());
 mcpApp.use(MCP_ENDPOINT, import_cors.default());
 mcpApp.all(MCP_ENDPOINT, async (req, res) => {
@@ -244914,3 +245179,12 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, () => {
 var mcpHttpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
   logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
 });
+function initProjects() {
+  projectStore.listProjects().then((projects) => {
+    projects.forEach((project) => {
+      projectStore.getProject(project.name, false).then((project2) => {
+        project2.listPackages();
+      });
+    });
+  });
+}

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.83",
+   "version": "0.0.85",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -21,21 +21,22 @@
       "generate-api-types": "bunx openapi-typescript ../../api-doc.yaml --output src/api.ts"
    },
    "dependencies": {
-      "@malloydata/db-bigquery": "^0.0.293",
-      "@malloydata/db-duckdb": "^0.0.293",
-      "@malloydata/db-mysql": "^0.0.293",
-      "@malloydata/db-postgres": "^0.0.293",
-      "@malloydata/db-snowflake": "^0.0.293",
-      "@malloydata/db-trino": "^0.0.293",
-      "@malloydata/malloy": "^0.0.293",
-      "@malloydata/malloy-sql": "^0.0.293",
-      "@malloydata/render": "^0.0.293",
+      "@malloydata/db-bigquery": "^0.0.295",
+      "@malloydata/db-duckdb": "^0.0.295",
+      "@malloydata/db-mysql": "^0.0.295",
+      "@malloydata/db-postgres": "^0.0.295",
+      "@malloydata/db-snowflake": "^0.0.295",
+      "@malloydata/db-trino": "^0.0.295",
+      "@malloydata/malloy": "^0.0.295",
+      "@malloydata/malloy-sql": "^0.0.295",
+      "@malloydata/render": "^0.0.295",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",
+      "async-mutex": "^0.5.0",
       "body-parser": "^1.20.2",
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",

package/src/controller/database.controller.ts

@@ -14,8 +14,8 @@ export class DatabaseController {
       projectName: string,
       packageName: string,
    ): Promise<ApiDatabase[]> {
-      const project = await this.projectStore.getProject(projectName);
-      const p = await project.getPackage(packageName);
+      const project = await this.projectStore.getProject(projectName, false);
+      const p = await project.getPackage(packageName, false);
       return p.listDatabases();
    }
 }

package/src/logger.ts

@@ -27,20 +27,19 @@ export const logAxiosError = (error: AxiosError) => {
    if (error.response) {
       // The request was made and the server responded with a status code
       // that falls out of the range of 2xx
-      logger.error("Axios error", {
-         data: error.response.data,
+      logger.error("Axios server-side error", {
+         url: error.response.config.url,
          status: error.response.status,
          headers: error.response.headers,
+         data: error.response.data,
       });
    } else if (error.request) {
       // The request was made but no response was received
       // `error.request` is an instance of XMLHttpRequest in the browser and an instance of
       // http.ClientRequest in node.js
-      logger.error("Failed to receive a response from the server", {
-         request: error.request,
-      });
+      logger.error("Axios client-side error", { error: error.request });
    } else {
       // Something happened in setting up the request that triggered an Error
-      logger.error("Error", { error });
+      logger.error("Axios unknown error", { error });
    }
 };

package/src/server.ts

@@ -92,6 +92,7 @@ const scheduleController = new ScheduleController(projectStore);
 
 const mcpApp = express();
 
+initProjects();
 mcpApp.use(MCP_ENDPOINT, express.json());
 mcpApp.use(MCP_ENDPOINT, cors());
 
@@ -634,3 +635,14 @@ const mcpHttpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
 });
 
 export { app, mainServer as httpServer, mcpApp, mcpHttpServer };
+
+// Warm up the packages
+function initProjects() {
+   projectStore.listProjects().then((projects) => {
+      projects.forEach((project) => {
+         projectStore.getProject(project.name!, false).then((project) => {
+            project.listPackages();
+         });
+      });
+   });
+}

package/src/service/connection.ts

@@ -91,7 +91,7 @@ export async function createConnections(basePath: string): Promise<{
                const config = {
                   host: connection.mysqlConnection.host,
                   port: connection.mysqlConnection.port,
-                  username: connection.mysqlConnection.user,
+                  user: connection.mysqlConnection.user,
                   password: connection.mysqlConnection.password,
                   database: connection.mysqlConnection.database,
                };

package/src/service/package.ts

@@ -31,6 +31,7 @@ type ApiSchedule = components["schemas"]["Schedule"];
 type ApiColumn = components["schemas"]["Column"];
 type ApiTableDescription = components["schemas"]["TableDescription"];
 
+const ENABLE_LIST_MODEL_COMPILATION = true;
 export class Package {
    private projectName: string;
    private packageName: string;
@@ -74,17 +75,43 @@ export class Package {
    ): Promise<Package> {
       const startTime = performance.now();
       await Package.validatePackageManifestExistsOrThrowError(packagePath);
+      const manifestValidationTime = performance.now();
+      logger.info("Package manifest validation completed", {
+         packageName,
+         duration: manifestValidationTime - startTime,
+         unit: "ms",
+      });
 
       try {
          const packageConfig = await Package.readPackageConfig(packagePath);
+         const packageConfigTime = performance.now();
+         logger.info("Package config read completed", {
+            packageName,
+            duration: packageConfigTime - manifestValidationTime,
+            unit: "ms",
+         });
          packageConfig.resource = `${API_PREFIX}/projects/${projectName}/packages/${packageName}`;
 
          const databases = await Package.readDatabases(packagePath);
+         const databasesTime = performance.now();
+         logger.info("Databases read completed", {
+            packageName,
+            databaseCount: databases.length,
+            duration: databasesTime - packageConfigTime,
+            unit: "ms",
+         });
          const connections = new Map<string, Connection>(projectConnections);
 
          // Package connections override project connections.
          const { malloyConnections: packageConnections } =
             await createConnections(packagePath);
+         const connectionsTime = performance.now();
+         logger.info("Package connections created", {
+            packageName,
+            connectionCount: packageConnections.size,
+            duration: connectionsTime - databasesTime,
+            unit: "ms",
+         });
          packageConnections.forEach((connection) => {
             connections.set(connection.name, connection);
          });
@@ -100,7 +127,20 @@ export class Package {
             packagePath,
             connections,
          );
+         const modelsTime = performance.now();
+         logger.info("Models loaded", {
+            packageName,
+            modelCount: models.size,
+            duration: modelsTime - connectionsTime,
+            unit: "ms",
+         });
          const scheduler = Scheduler.create(models);
+         const schedulerTime = performance.now();
+         logger.info("Scheduler created", {
+            packageName,
+            duration: schedulerTime - modelsTime,
+            unit: "ms",
+         });
          const endTime = performance.now();
          const executionTime = endTime - startTime;
          this.packageLoadHistogram.record(executionTime, {
@@ -117,14 +157,16 @@ export class Package {
             scheduler,
          );
       } catch (error) {
-         logger.error("Error loading package", { error });
+         logger.error(`Error loading package ${packageName}`, { error });
          const endTime = performance.now();
          const executionTime = endTime - startTime;
          this.packageLoadHistogram.record(executionTime, {
             malloy_package_name: packageName,
             status: "error",
          });
-         throw new Error("Error loading package: " + error);
+         throw new Error(`Error loading package ${packageName}`, {
+            cause: error,
+         });
       }
    }
 
@@ -164,13 +206,15 @@ export class Package {
             })
             .map(async (modelPath) => {
                let error: string | undefined;
-               try {
-                  await this.models.get(modelPath)?.getModel();
-               } catch (modelError) {
-                  error =
-                     modelError instanceof Error
-                        ? modelError.message
-                        : undefined;
+               if (ENABLE_LIST_MODEL_COMPILATION) {
+                  try {
+                     await this.models.get(modelPath)?.getModel();
+                  } catch (modelError) {
+                     error =
+                        modelError instanceof Error
+                           ? modelError.message
+                           : undefined;
+                  }
                }
                return {
                   projectName: this.projectName,
@@ -190,7 +234,10 @@ export class Package {
                return modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
             })
             .map(async (modelPath) => {
-               const error = this.models.get(modelPath)?.getNotebookError();
+               let error: Error | undefined;
+               if (ENABLE_LIST_MODEL_COMPILATION) {
+                  error = this.models.get(modelPath)?.getNotebookError();
+               }
                return {
                   projectName: this.projectName,
                   packageName: this.packageName,