@malloy-publisher/server 0.0.157 → 0.0.159

package/dist/instrumentation.js

@@ -116693,7 +116693,20 @@ var import_sdk_logs = __toESM(require_src7());
 // src/logger.ts
 var import_winston = __toESM(require_winston());
 var isTelemetryEnabled = Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+var VALID_LOG_LEVELS = ["error", "warn", "info", "verbose", "debug", "silly"];
+var getLogLevel = () => {
+  if (process.env.LOG_LEVEL) {
+    const logLevel = process.env.LOG_LEVEL.toLowerCase();
+    if (VALID_LOG_LEVELS.includes(logLevel)) {
+      return logLevel;
+    } else {
+      console.error(`Invalid log level: ${process.env.LOG_LEVEL}. Valid log levels are: ${VALID_LOG_LEVELS.join(", ")}. Defaulting to "debug".`);
+    }
+  }
+  return "debug";
+};
 var logger = import_winston.default.createLogger({
+  level: getLogLevel(),
   format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.metadata({
     fillExcept: ["message", "level", "timestamp"]
   }), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
@@ -116710,6 +116723,14 @@ function extractTraceIdFromTraceparent(traceparent) {
   }
   return;
 }
+var DISABLE_RESPONSE_LOGGING = process.env.DISABLE_RESPONSE_LOGGING === "true" || process.env.DISABLE_RESPONSE_LOGGING === "1";
+function formatDuration(durationMs) {
+  if (durationMs >= 1000) {
+    const seconds = durationMs / 1000;
+    return `${seconds.toFixed(2)}s`;
+  }
+  return `${durationMs.toFixed(2)}ms`;
+}
 var loggerMiddleware = (req, res, next) => {
   const startTime = performance.now();
   const resJson = res.json;
@@ -116719,16 +116740,19 @@ var loggerMiddleware = (req, res, next) => {
   };
   res.on("finish", () => {
     const endTime = performance.now();
+    const durationMs = endTime - startTime;
     const traceparent = req.headers["traceparent"];
     const traceId = extractTraceIdFromTraceparent(traceparent);
     const logMetadata = {
       statusCode: res.statusCode,
-      duration: endTime - startTime,
+      duration: formatDuration(durationMs),
       payload: req.body,
-      response: res.locals.body,
       params: req.params,
       query: req.query
     };
+    if (!DISABLE_RESPONSE_LOGGING) {
+      logMetadata.response = res.locals.body;
+    }
     if (traceId) {
       logMetadata.traceId = traceId;
     }

package/dist/server.js

@@ -125120,7 +125120,20 @@ class FrozenConfigError extends Error {
 // src/logger.ts
 var import_winston = __toESM(require_winston());
 var isTelemetryEnabled = Boolean(process.env.OTEL_EXPORTER_OTLP_ENDPOINT);
+var VALID_LOG_LEVELS = ["error", "warn", "info", "verbose", "debug", "silly"];
+var getLogLevel = () => {
+  if (process.env.LOG_LEVEL) {
+    const logLevel = process.env.LOG_LEVEL.toLowerCase();
+    if (VALID_LOG_LEVELS.includes(logLevel)) {
+      return logLevel;
+    } else {
+      console.error(`Invalid log level: ${process.env.LOG_LEVEL}. Valid log levels are: ${VALID_LOG_LEVELS.join(", ")}. Defaulting to "debug".`);
+    }
+  }
+  return "debug";
+};
 var logger = import_winston.default.createLogger({
+  level: getLogLevel(),
   format: isTelemetryEnabled ? import_winston.default.format.combine(import_winston.default.format.uncolorize(), import_winston.default.format.timestamp(), import_winston.default.format.metadata({
     fillExcept: ["message", "level", "timestamp"]
   }), import_winston.default.format.json()) : import_winston.default.format.combine(import_winston.default.format.colorize(), import_winston.default.format.simple()),
@@ -125137,6 +125150,14 @@ function extractTraceIdFromTraceparent(traceparent) {
   }
   return;
 }
+var DISABLE_RESPONSE_LOGGING = process.env.DISABLE_RESPONSE_LOGGING === "true" || process.env.DISABLE_RESPONSE_LOGGING === "1";
+function formatDuration(durationMs) {
+  if (durationMs >= 1000) {
+    const seconds = durationMs / 1000;
+    return `${seconds.toFixed(2)}s`;
+  }
+  return `${durationMs.toFixed(2)}ms`;
+}
 var loggerMiddleware = (req, res, next) => {
   const startTime = performance.now();
   const resJson = res.json;
@@ -125146,16 +125167,19 @@ var loggerMiddleware = (req, res, next) => {
   };
   res.on("finish", () => {
     const endTime = performance.now();
+    const durationMs = endTime - startTime;
     const traceparent = req.headers["traceparent"];
     const traceId = extractTraceIdFromTraceparent(traceparent);
     const logMetadata = {
       statusCode: res.statusCode,
-      duration: endTime - startTime,
+      duration: formatDuration(durationMs),
       payload: req.body,
-      response: res.locals.body,
       params: req.params,
       query: req.query
     };
+    if (!DISABLE_RESPONSE_LOGGING) {
+      logMetadata.response = res.locals.body;
+    }
     if (traceId) {
       logMetadata.traceId = traceId;
     }
@@ -130122,7 +130146,6 @@ class ModelController {
 
 // src/controller/package.controller.ts
 var path2 = __toESM(require("path"));
-
 class PackageController {
   projectStore;
   constructor(projectStore) {
@@ -136043,6 +136066,7 @@ class DuckDBConnection2 {
   db = null;
   connection = null;
   dbPath;
+  mutex = new Mutex;
   constructor(dbPath) {
     this.dbPath = dbPath || path4.join(process.cwd(), "publisher.db");
   }
@@ -136099,13 +136123,15 @@ class DuckDBConnection2 {
   async isInitialized() {
     if (!this.connection)
       return false;
-    return new Promise((resolve3) => {
-      this.connection.all("SELECT name FROM sqlite_master WHERE type='table' AND name='projects'", (err, rows) => {
-        if (err) {
-          resolve3(false);
-          return;
-        }
-        resolve3(rows && rows.length > 0);
+    return this.mutex.runExclusive(async () => {
+      return new Promise((resolve3) => {
+        this.connection.all("SELECT name FROM sqlite_master WHERE type='table' AND name='projects'", (err, rows) => {
+          if (err) {
+            resolve3(false);
+            return;
+          }
+          resolve3(rows && rows.length > 0);
+        });
       });
     });
   }
@@ -136113,40 +136139,44 @@ class DuckDBConnection2 {
     if (!this.connection) {
       throw new Error("Database not initialized");
     }
-    return new Promise((resolve3, reject) => {
-      const callback = (err) => {
-        if (err) {
-          reject(new Error(`Query execution failed: ${err.message}
+    return this.mutex.runExclusive(async () => {
+      return new Promise((resolve3, reject) => {
+        const callback = (err) => {
+          if (err) {
+            reject(new Error(`Query execution failed: ${err.message}
 Query: ${query}`));
-          return;
+            return;
+          }
+          resolve3();
+        };
+        if (params && params.length > 0) {
+          this.connection.run(query, ...params, callback);
+        } else {
+          this.connection.run(query, callback);
         }
-        resolve3();
-      };
-      if (params && params.length > 0) {
-        this.connection.run(query, ...params, callback);
-      } else {
-        this.connection.run(query, callback);
-      }
+      });
     });
   }
   async all(query, params) {
     if (!this.connection) {
       throw new Error("Database not initialized");
     }
-    return new Promise((resolve3, reject) => {
-      const callback = (err, rows) => {
-        if (err) {
-          reject(new Error(`Query execution failed: ${err.message}
+    return this.mutex.runExclusive(async () => {
+      return new Promise((resolve3, reject) => {
+        const callback = (err, rows) => {
+          if (err) {
+            reject(new Error(`Query execution failed: ${err.message}
 Query: ${query}`));
-          return;
+            return;
+          }
+          resolve3(rows || []);
+        };
+        if (params && params.length > 0) {
+          this.connection.all(query, ...params, callback);
+        } else {
+          this.connection.all(query, callback);
         }
-        resolve3(rows || []);
-      };
-      if (params && params.length > 0) {
-        this.connection.all(query, ...params, callback);
-      } else {
-        this.connection.all(query, callback);
-      }
+      });
     });
   }
   async get(query, params) {
@@ -136750,6 +136780,7 @@ class Model {
   modelInfo;
   sources;
   queries;
+  sourceInfos;
   runnableNotebookCells;
   compilationError;
   meter = import_api.metrics.getMeter("publisher");
@@ -136757,7 +136788,7 @@ class Model {
     description: "How long it takes to execute a Malloy model query",
     unit: "ms"
   });
-  constructor(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, runnableNotebookCells, compilationError) {
+  constructor(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos, runnableNotebookCells, compilationError) {
     this.packageName = packageName;
     this.modelPath = modelPath;
     this.dataStyles = dataStyles;
@@ -136766,6 +136797,7 @@ class Model {
     this.modelMaterializer = modelMaterializer;
     this.sources = sources;
     this.queries = queries;
+    this.sourceInfos = sourceInfos;
     this.runnableNotebookCells = runnableNotebookCells;
     this.compilationError = compilationError;
     this.modelInfo = this.modelDef ? import_malloy2.modelDefToModelInfo(this.modelDef) : undefined;
@@ -136777,12 +136809,41 @@ class Model {
       let modelDef = undefined;
       let sources = undefined;
       let queries = undefined;
+      const sourceInfos = [];
       if (modelMaterializer) {
         modelDef = (await modelMaterializer.getModel())._modelDef;
         sources = Model.getSources(modelPath, modelDef);
         queries = Model.getQueries(modelPath, modelDef);
+        const imports = modelDef.imports || [];
+        const importedSourceNames = new Set;
+        for (const importLocation of imports) {
+          try {
+            const modelString = await runtime.urlReader.readURL(new URL(importLocation.importURL));
+            const importedModelDef = (await runtime.loadModel(modelString, { importBaseURL }).getModel())._modelDef;
+            const importedModelInfo = import_malloy2.modelDefToModelInfo(importedModelDef);
+            const importedSources = importedModelInfo.entries.filter((entry) => entry.kind === "source");
+            for (const source of importedSources) {
+              if (!importedSourceNames.has(source.name)) {
+                sourceInfos.push(source);
+                importedSourceNames.add(source.name);
+              }
+            }
+          } catch (importError) {
+            logger.warn("Failed to load sourceInfo from import", {
+              importURL: importLocation.importURL,
+              error: importError
+            });
+          }
+        }
+        const localModelInfo = import_malloy2.modelDefToModelInfo(modelDef);
+        const localSources = localModelInfo.entries.filter((entry) => entry.kind === "source");
+        for (const source of localSources) {
+          if (!importedSourceNames.has(source.name)) {
+            sourceInfos.push(source);
+          }
+        }
       }
-      return new Model(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, runnableNotebookCells, undefined);
+      return new Model(packageName, modelPath, dataStyles, modelType, modelMaterializer, modelDef, sources, queries, sourceInfos.length > 0 ? sourceInfos : undefined, runnableNotebookCells, undefined);
     } catch (error) {
       let computedError = error;
       if (error instanceof Error && error.stack) {
@@ -136795,7 +136856,7 @@ class Model {
         }
         computedError = new ModelCompilationError(error);
       }
-      return new Model(packageName, modelPath, dataStyles, modelType, undefined, undefined, undefined, undefined, undefined, computedError);
+      return new Model(packageName, modelPath, dataStyles, modelType, undefined, undefined, undefined, undefined, undefined, undefined, computedError);
     }
   }
   getPath() {
@@ -136808,9 +136869,7 @@ class Model {
     return this.sources;
   }
   getSourceInfos() {
-    return this.modelDef ? import_malloy2.modelDefToModelInfo(this.modelDef).entries.filter((entry) => {
-      return entry.kind === "source";
-    }) : undefined;
+    return this.sourceInfos;
   }
   getQueries() {
     return this.sources;
@@ -137035,8 +137094,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`);
     const modelConnections = new Map;
     for (const [name, connection] of connections.entries()) {
       const isDuckDB = connection instanceof import_db_duckdb2.DuckDBConnection || connection.constructor.name === "DuckDBConnection";
-      if (isDuckDB) {
-        const modelDuckDBConnection = new import_db_duckdb2.DuckDBConnection(name, ":memory:", workingDirectory);
+      if (isDuckDB && connection.workingDirectory !== workingDirectory) {
+        const databasePath = connection.databasePath || ":memory:";
+        const modelDuckDBConnection = new import_db_duckdb2.DuckDBConnection(name, databasePath, workingDirectory);
         modelConnections.set(name, modelDuckDBConnection);
       } else {
         modelConnections.set(name, connection);
@@ -137226,16 +137286,14 @@ class Package {
     const manifestValidationTime = performance.now();
     logger.info("Package manifest validation completed", {
       packageName,
-      duration: manifestValidationTime - startTime,
-      unit: "ms"
+      duration: formatDuration(manifestValidationTime - startTime)
     });
     try {
       const packageConfig = await Package.readPackageConfig(packagePath);
       const packageConfigTime = performance.now();
       logger.info("Package config read completed", {
         packageName,
-        duration: packageConfigTime - manifestValidationTime,
-        unit: "ms"
+        duration: formatDuration(packageConfigTime - manifestValidationTime)
       });
       packageConfig.resource = `${API_PREFIX}/projects/${projectName}/packages/${packageName}`;
       const databases = await Package.readDatabases(packagePath);
@@ -137243,8 +137301,7 @@ class Package {
       logger.info("Databases read completed", {
         packageName,
         databaseCount: databases.length,
-        duration: databasesTime - packageConfigTime,
-        unit: "ms"
+        duration: formatDuration(databasesTime - packageConfigTime)
       });
       const connections = new Map(projectConnections);
       const duckdbConnections = await createPackageDuckDBConnections(packageConnections, packagePath);
@@ -137256,8 +137313,7 @@ class Package {
       logger.info("Models loaded", {
         packageName,
         modelCount: models.size,
-        duration: modelsTime - databasesTime,
-        unit: "ms"
+        duration: formatDuration(modelsTime - databasesTime)
       });
       for (const [modelPath, model] of models.entries()) {
         const maybeModel = model;
@@ -137284,8 +137340,7 @@ class Package {
       });
       logger.info(`Successfully loaded package ${packageName}`, {
         packageName,
-        duration: executionTime,
-        unit: "ms"
+        duration: formatDuration(executionTime)
       });
       return new Package(projectName, packageName, packagePath, packageConfig, databases, models, connections);
     } catch (error) {
@@ -137297,6 +137352,17 @@ class Package {
         malloy_package_name: packageName,
         status: "error"
       });
+      try {
+        await fs5.rm(packagePath, {
+          recursive: true,
+          force: true
+        });
+        logger.info(`Cleaned up failed package directory: ${packagePath}`);
+      } catch (cleanupError) {
+        logger.warn(`Failed to clean up package directory ${packagePath}`, {
+          error: cleanupError
+        });
+      }
       throw error;
     }
   }
@@ -137708,17 +137774,31 @@ class Project {
     this.apiConnections = apiConnections;
   }
   deleteConnection(connectionName) {
-    const deleted = this.malloyConnections.delete(connectionName);
+    this.malloyConnections.get(connectionName)?.close();
+    const isDeleted = this.malloyConnections.delete(connectionName);
     const index = this.apiConnections.findIndex((conn) => conn.name === connectionName);
     if (index !== -1) {
       this.apiConnections.splice(index, 1);
     }
-    if (deleted || index !== -1) {
+    if (isDeleted || index !== -1) {
       logger.info(`Removed connection ${connectionName} from project ${this.projectName}`);
     } else {
       logger.warn(`Connection ${connectionName} not found in project ${this.projectName}`);
     }
   }
+  closeAllConnections() {
+    for (const [connectionName, connection] of this.malloyConnections) {
+      try {
+        connection.close();
+        logger.info(`Closed connection ${connectionName} for project ${this.projectName}`);
+      } catch (error) {
+        logger.error(`Error closing connection ${connectionName} for project ${this.projectName}`, { error });
+      }
+    }
+    this.malloyConnections.clear();
+    this.apiConnections = [];
+    logger.info(`Closed all connections for project ${this.projectName}`);
+  }
   async serialize() {
     return {
       ...this.metadata,
@@ -137820,7 +137900,8 @@ class ProjectStore {
         }
       }
       this.isInitialized = true;
-      logger.info(`Project store successfully initialized in ${performance.now() - initialTime}ms`);
+      const initializationDuration = performance.now() - initialTime;
+      logger.info(`Project store successfully initialized in ${formatDuration(initializationDuration)}`);
     } catch (error) {
       logger.error("Error initializing project store", { error });
       process.exit(1);
@@ -138192,6 +138273,7 @@ class ProjectStore {
       return;
     }
     const projectPath = project.metadata?.location;
+    project.closeAllConnections();
     this.projects.delete(projectName);
     await this.deleteProjectFromDatabase(projectName);
     if (projectPath) {
@@ -142492,7 +142574,7 @@ function initializeMcpServer(projectStore) {
   registerPromptCapability(mcpServer, projectStore);
   const endTime = performance.now();
   logger.info(`[MCP Init] Finished initializeMcpServer`, {
-    duration: endTime - startTime
+    duration: formatDuration(endTime - startTime)
   });
   return mcpServer;
 }
@@ -142770,7 +142852,6 @@ app.get(`${API_PREFIX2}/projects/:projectName/connections/:connectionName/schema
   logger.info("req.params", { params: req.params });
   try {
     const results = await connectionController.listTables(req.params.projectName, req.params.connectionName, req.params.schemaName);
-    logger.info("results", { results });
     res.status(200).json(results);
   } catch (error) {
     logger.error(error);
@@ -142782,7 +142863,6 @@ app.get(`${API_PREFIX2}/projects/:projectName/connections/:connectionName/schema
   logger.info("req.params", { params: req.params });
   try {
     const results = await connectionController.getTable(req.params.projectName, req.params.connectionName, req.params.schemaName, req.params.tablePath);
-    logger.info("results", { results });
     res.status(200).json(results);
   } catch (error) {
     logger.error(error);

package/k6-tests/README.md

@@ -0,0 +1,88 @@
+# k6-tests
+
+Performance and load testing suite for the Malloy Publisher API using k6.
+
+## Installation
+
+To install dependencies:
+
+```bash
+bun install
+```
+
+## Available Tests
+
+### Smoke Test
+
+Basic functionality verification under minimal load:
+
+```bash
+bun run smoke-test
+```
+
+### Load Tests
+
+Individual CRUD operation load tests:
+
+```bash
+bun run load-test-projects      # Projects CRUD load test
+bun run load-test-connections   # Connections CRUD load test
+bun run load-test-packages      # Packages CRUD load test
+```
+
+Combined load test (runs all three):
+
+```bash
+bun run load-test
+```
+
+### Performance Tests
+
+```bash
+bun run spike-test        # Spike test - sudden traffic spikes
+bun run stress-test       # Stress test - beyond normal capacity
+bun run breakpoint-test   # Breakpoint test - find maximum capacity
+bun run soak-test         # Soak test - sustained load over time
+```
+
+### Run All Tests
+
+```bash
+bun run all-tests
+```
+
+## Environment Variables
+
+- `PUBLISHER_URL` - Base URL of the publisher service (default: `http://localhost:4000`)
+- `PROJECT_NAME` - Project name for tests (default: `malloy-samples`)
+- `AUTH_TOKEN` - Authorization token for API requests
+- `GOOGLE_APPLICATION_CREDENTIALS` - BigQuery service account JSON (for BigQuery tests)
+- `USE_VERSION_ID` - Set to `"true"` to use version IDs in API calls
+- `MAX_VIEWS_PER_MODEL` - Maximum views to test per model (default: `5`)
+- `DEBUG` - Set to `"true"` for verbose logging
+- `WHITELISTED_PACKAGES` - JSON array of package names to test
+- `AVAILABLE_PACKAGES` - JSON array of available package names
+
+## Client Generation
+
+To regenerate API clients from OpenAPI spec:
+
+```bash
+bun run generate-clients
+```
+
+## Cleanup
+
+Remove generated files and dependencies:
+
+```bash
+bun run clean
+```
+
+## Build
+
+Compile TypeScript:
+
+```bash
+bun run build
+```

package/k6-tests/breakpoint-test.ts

@@ -1,5 +1,6 @@
 import { check, sleep } from "k6";
 import {
+   getAvailablePackages,
    getModelData,
    getModels,
    getPackages,
@@ -39,7 +40,11 @@ export const breakpointTest: TestPreset = {
    },
    run: () => {
       const packages = getPackages();
+      const availablePackages = getAvailablePackages();
       for (const pkg of packages) {
+         if (!availablePackages.includes(pkg.name)) {
+            continue;
+         }
          for (const model of getModels(pkg.name)) {
             const modelData = getModelData(pkg.name, model.path);
             for (const view of getViews(modelData)) {