@malloy-publisher/server 0.0.165 → 0.0.168

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.165",
+   "version": "0.0.168",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -42,6 +42,7 @@
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",
+      "@opentelemetry/exporter-prometheus": "^0.212.0",
       "@opentelemetry/sdk-metrics": "^2.0.0",
       "@opentelemetry/sdk-node": "^0.200.0",
       "@opentelemetry/sdk-trace-node": "^2.0.0",

package/src/controller/compile.controller.ts

@@ -0,0 +1,35 @@
+import type { LogMessage } from "@malloydata/malloy";
+import { ProjectStore } from "../service/project_store";
+
+export class CompileController {
+   private projectStore: ProjectStore;
+
+   constructor(projectStore: ProjectStore) {
+      this.projectStore = projectStore;
+   }
+
+   public async compile(
+      projectName: string,
+      packageName: string,
+      modelName: string,
+      source: string,
+      includeSql: boolean = false,
+   ): Promise<{ status: string; problems: LogMessage[]; sql?: string }> {
+      const project = await this.projectStore.getProject(projectName, false);
+      const { problems, sql } = await project.compileSource(
+         packageName,
+         modelName,
+         source,
+         includeSql,
+      );
+
+      // Determine overall status based on presence of errors
+      const hasErrors = problems.some((p) => p.severity === "error");
+
+      return {
+         status: hasErrors ? "error" : "success",
+         problems: problems,
+         ...(sql !== undefined && { sql }),
+      };
+   }
+}

package/src/controller/connection.controller.ts

@@ -155,17 +155,37 @@ export class ConnectionController {
    public async getTable(
       projectName: string,
       connectionName: string,
-      _schemaName: string,
+      schemaName: string,
       tablePath: string,
    ): Promise<ApiTable> {
       const malloyConnection = await this.getMalloyConnection(
          projectName,
          connectionName,
       );
+      const connection = await this.getConnection(projectName, connectionName);
+
+      if (connection.type === "ducklake") {
+         if (tablePath.split(".").length === 1) {
+            // tablePath is just the table name, construct full path
+            tablePath = `${connectionName}.${schemaName}.${tablePath}`;
+         } else if (
+            tablePath.split(".").length === 2 &&
+            !tablePath.startsWith(connectionName)
+         ) {
+            // tablePath is schemaName.tableName but missing connection prefix
+            tablePath = `${connectionName}.${tablePath}`;
+         }
+         // If tablePath already has 3+ parts or starts with connection name, use as-is
+      }
+
+      const tableKey = tablePath.split(".").pop();
+      if (!tableKey) {
+         throw new Error(`Invalid tablePath: ${tablePath}`);
+      }
 
       const tableSource = await getConnectionTableSource(
          malloyConnection,
-         tablePath.split(".").pop()!, // tableKey is the table name
+         tableKey, // tableKey is the table name
          tablePath,
       );
 

package/src/controller/model.controller.ts

@@ -38,16 +38,27 @@ export class ModelController {
       packageName: string,
       modelPath: string,
    ): Promise<ApiCompiledModel> {
-      const project = await this.projectStore.getProject(projectName, false);
-      const p = await project.getPackage(packageName, false);
-      const model = p.getModel(modelPath);
-      if (!model) {
-         throw new ModelNotFoundError(`${modelPath} does not exist`);
-      }
-      if (model.getType() === "notebook") {
-         throw new ModelNotFoundError(`${modelPath} is a notebook`);
+      try {
+         const project = await this.projectStore.getProject(projectName, false);
+         const p = await project.getPackage(packageName, false);
+         const model = p.getModel(modelPath);
+         if (!model) {
+            throw new ModelNotFoundError(`${modelPath} does not exist`);
+         }
+         if (model.getType() === "notebook") {
+            throw new ModelNotFoundError(`${modelPath} is a notebook`);
+         }
+         return await model.getModel();
+      } catch (error) {
+         // Re-throw ModelNotFoundError as-is
+         if (error instanceof ModelNotFoundError) {
+            throw error;
+         }
+         // Wrap other errors with more context
+         throw new Error(
+            `Failed to get model ${modelPath} from package ${packageName} in project ${projectName}: ${error}`,
+         );
       }
-      return model.getModel();
    }
 
    public async getNotebook(

package/src/health.ts

@@ -13,6 +13,7 @@
 import { Express, NextFunction, Request, Response } from "express";
 import { Server } from "http";
 import { components } from "./api";
+import { shutdownSDK } from "./instrumentation";
 import { logger } from "./logger";
 export type OperationalState =
    components["schemas"]["ServerStatus"]["operationalState"];
@@ -112,6 +113,13 @@ export function registerSignalHandlers(
          closeServer(mcpServer, "MCP server"),
       ]);
 
+      try {
+         await shutdownSDK();
+         logger.info("OpenTelemetry SDK shut down");
+      } catch (_error) {
+         /* do nothing */
+      }
+
       try {
          logger.close();
       } catch (_error) {

package/src/instrumentation.ts

@@ -1,59 +1,148 @@
-import { NodeSDK } from "@opentelemetry/sdk-node";
-import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
-import { OTLPMetricExporter } from "@opentelemetry/exporter-metrics-otlp-proto";
-import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
+import { metrics } from "@opentelemetry/api";
 import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { OTLPLogExporter } from "@opentelemetry/exporter-logs-otlp-proto";
+import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-proto";
+import {
+   ExpressInstrumentation,
+   ExpressLayerType,
+} from "@opentelemetry/instrumentation-express";
 import { resourceFromAttributes } from "@opentelemetry/resources";
+import { BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
 import {
    ATTR_SERVICE_NAME,
    ATTR_SERVICE_VERSION,
 } from "@opentelemetry/semantic-conventions";
-import { PeriodicExportingMetricReader } from "@opentelemetry/sdk-metrics";
-import { BatchSpanProcessor } from "@opentelemetry/sdk-trace-base";
-import { BatchLogRecordProcessor } from "@opentelemetry/sdk-logs";
+import type { NextFunction, Request, Response } from "express";
 import { logger } from "./logger";
 
+let prometheusExporter: PrometheusExporter | null = null;
+let sdk: NodeSDK | null = null;
+
+export function getPrometheusMetricsHandler() {
+   if (!prometheusExporter) {
+      throw new Error("Prometheus exporter not initialized");
+   }
+   return (req: Request, res: Response) => {
+      prometheusExporter!.getMetricsRequestHandler(req, res);
+   };
+}
+
+/**
+ * Shuts down the OpenTelemetry SDK gracefully.
+ * Should be called during application shutdown.
+ */
+export async function shutdownSDK(): Promise<void> {
+   if (sdk) {
+      await sdk.shutdown();
+      sdk = null;
+   }
+}
+
 function instrument() {
    const otelCollectorUrl = process.env.OTEL_EXPORTER_OTLP_ENDPOINT;
-   if (!otelCollectorUrl) {
-      logger.info("No OTLP collector URL found, skipping instrumentation");
-      return;
-   }
 
-   logger.info(
-      `Initializing OpenTelemetry SDK with OTLP collector at ${otelCollectorUrl}`,
-   );
-   const traceExporter = new OTLPTraceExporter({
-      url: `${otelCollectorUrl}/v1/traces`,
-      headers: {},
-   });
-   const metricExporter = new OTLPMetricExporter({
-      url: `${otelCollectorUrl}/v1/metrics`,
-      headers: {},
-   });
-   const logExporter = new OTLPLogExporter({
-      url: `${otelCollectorUrl}/v1/logs`,
-      headers: {},
+   prometheusExporter = new PrometheusExporter({
+      preventServerStart: true,
    });
 
-   const sdk = new NodeSDK({
-      serviceName: "publisher",
+   const instrumentations = [
+      getNodeAutoInstrumentations(),
+      new ExpressInstrumentation({
+         ignoreLayersType: [ExpressLayerType.MIDDLEWARE],
+         ignoreLayers: [/\/health/, /\/metrics/],
+      }),
+   ];
+
+   sdk = new NodeSDK({
       resource: resourceFromAttributes({
          [ATTR_SERVICE_NAME]: "publisher",
          [ATTR_SERVICE_VERSION]: "1.0.0",
       }),
       autoDetectResources: true,
-      traceExporter: traceExporter,
-      metricReader: new PeriodicExportingMetricReader({
-         exporter: metricExporter,
+      metricReader: prometheusExporter,
+      instrumentations,
+      ...(otelCollectorUrl && {
+         spanProcessors: [
+            new BatchSpanProcessor(
+               new OTLPTraceExporter({
+                  url: `${otelCollectorUrl}/v1/traces`,
+               }),
+            ),
+         ],
+         logRecordProcessors: [
+            new BatchLogRecordProcessor(
+               new OTLPLogExporter({
+                  url: `${otelCollectorUrl}/v1/logs`,
+               }),
+            ),
+         ],
       }),
-      instrumentations: [getNodeAutoInstrumentations()],
-      spanProcessors: [new BatchSpanProcessor(traceExporter)],
-      logRecordProcessors: [new BatchLogRecordProcessor(logExporter)],
    });
 
    sdk.start();
-   logger.info("Initialized OpenTelemetry SDK");
+
+   if (otelCollectorUrl) {
+      logger.info(
+         `OpenTelemetry SDK initialized with OTLP collector at ${otelCollectorUrl}`,
+      );
+   } else {
+      logger.info("OpenTelemetry SDK initialized with Prometheus metrics only");
+   }
 }
 
 instrument();
+
+// --- HTTP metrics middleware ---
+
+const meter = metrics.getMeter("publisher");
+
+const httpRequestDuration = meter.createHistogram(
+   "http_server_request_duration_ms",
+   {
+      description: "Duration of HTTP requests in milliseconds",
+      unit: "ms",
+      advice: {
+         explicitBucketBoundaries: [
+            5, 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 60000,
+         ],
+      },
+   },
+);
+
+const httpRequestCount = meter.createCounter("http_server_requests_total", {
+   description: "Total number of HTTP requests",
+});
+
+const IGNORED_PATHS = new Set([
+   "/health",
+   "/health/liveness",
+   "/health/readiness",
+   "/metrics",
+]);
+
+export function httpMetricsMiddleware(
+   req: Request,
+   res: Response,
+   next: NextFunction,
+) {
+   const start = performance.now();
+
+   res.on("finish", () => {
+      if (IGNORED_PATHS.has(req.path)) return;
+
+      const duration = performance.now() - start;
+      const attrs = {
+         "http.method": req.method,
+         "http.route": req.route?.path ?? req.path,
+         "http.status_code": res.statusCode,
+      };
+
+      httpRequestDuration.record(duration, attrs);
+      httpRequestCount.add(1, attrs);
+   });
+
+   next();
+}

package/src/server.ts

@@ -1,3 +1,10 @@
+// Pre-load the instrumentation module; the instrumentation module must be loaded before the other imports.
+import "./instrumentation";
+import {
+   getPrometheusMetricsHandler,
+   httpMetricsMiddleware,
+} from "./instrumentation";
+
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
 import * as bodyParser from "body-parser";
 import cors from "cors";
@@ -6,6 +13,7 @@ import * as http from "http";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
+import { CompileController } from "./controller/compile.controller";
 import { ConnectionController } from "./controller/connection.controller";
 import { DatabaseController } from "./controller/database.controller";
 import { ModelController } from "./controller/model.controller";
@@ -19,6 +27,7 @@ import {
    registerSignalHandlers,
 } from "./health";
 import { logger, loggerMiddleware } from "./logger";
+
 import { initializeMcpServer } from "./mcp/server";
 import { ProjectStore } from "./service/project_store";
 // Parse command line arguments
@@ -110,7 +119,7 @@ const isDevelopment = process.env["NODE_ENV"] === "development";
 
 const app = express();
 app.use(loggerMiddleware);
-
+app.use(httpMetricsMiddleware);
 const projectStore = new ProjectStore(SERVER_ROOT);
 const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
@@ -118,9 +127,13 @@ const modelController = new ModelController(projectStore);
 const packageController = new PackageController(projectStore);
 const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
+const compileController = new CompileController(projectStore);
 
 export const mcpApp = express();
 
+// Register health endpoints on mcpApp (for E2E tests)
+registerHealthEndpoints(mcpApp);
+
 mcpApp.use(MCP_ENDPOINT, express.json());
 mcpApp.use(MCP_ENDPOINT, cors());
 
@@ -220,7 +233,10 @@ if (!isDevelopment) {
          target: "http://localhost:5173",
          changeOrigin: true,
          ws: true,
-         pathFilter: (path) => !path.startsWith("/api/"),
+         pathFilter: (path) =>
+            !path.startsWith("/api/") &&
+            !path.startsWith("/metrics") &&
+            !path.startsWith("/health"),
       }),
    );
 }
@@ -240,9 +256,19 @@ app.use(
 );
 app.use(bodyParser.json());
 
-// Register health check endpoints
+// Register health check endpoints on main app:
+// - Required for production/Kubernetes monitoring (main server on PUBLISHER_PORT)
 registerHealthEndpoints(app);
 
+// Register Prometheus metrics endpoint
+try {
+   const metricsHandler = getPrometheusMetricsHandler();
+   app.get("/metrics", metricsHandler);
+   logger.info("Prometheus metrics endpoint registered at /metrics");
+} catch (error) {
+   logger.warn("Failed to register Prometheus metrics endpoint", { error });
+}
+
 // Register draining guard middleware - must be after health endpoints but before other routes
 app.use(drainingGuard);
 
@@ -897,6 +923,26 @@ app.get(
    },
 );
 
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/models/:modelName/compile`,
+   async (req, res) => {
+      try {
+         const result = await compileController.compile(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.modelName,
+            req.body.source,
+            req.body.includeSql === true,
+         );
+         res.status(200).json(result);
+      } catch (error) {
+         logger.error("Compilation error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));