@malloy-publisher/server 0.0.181 → 0.0.183-dev

package/src/server.ts

@@ -6,13 +6,14 @@ import {
 } from "./instrumentation";
 
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
-import * as bodyParser from "body-parser";
+import bodyParser from "body-parser";
 import cors from "cors";
 import express from "express";
 import * as http from "http";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
+import { fileURLToPath } from "url";
 import { CompileController } from "./controller/compile.controller";
 import { ConnectionController } from "./controller/connection.controller";
 import { DatabaseController } from "./controller/database.controller";
@@ -20,7 +21,11 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { WatchModeController } from "./controller/watch-mode.controller";
-import { internalErrorToHttpError, NotImplementedError } from "./errors";
+import {
+   BadRequestError,
+   internalErrorToHttpError,
+   NotImplementedError,
+} from "./errors";
 import {
    drainingGuard,
    registerHealthEndpoints,
@@ -28,7 +33,11 @@ import {
 } from "./health";
 import { logger, loggerMiddleware } from "./logger";
 
+import { ManifestController } from "./controller/manifest.controller";
+import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { ManifestService } from "./service/manifest_service";
+import { MaterializationService } from "./service/materialization_service";
 import { ProjectStore } from "./service/project_store";
 
 /** Normalize an Express query param into a string[] or undefined. */
@@ -112,30 +121,37 @@ const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
 const SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(
    process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0,
 );
-// Find the app directory - handle NPX vs local execution
-let ROOT: string;
-if (require.main) {
-   // Use the main module's directory (works for NPX and direct execution)
-   ROOT = path.join(path.dirname(require.main.filename), "app");
-} else {
-   // Fallback to current script directory
-   ROOT = path.join(path.dirname(process.argv[1] || __filename), "app");
-}
+// Find the app directory relative to this bundled server file.
+// Works under both ESM (import.meta.url) and when invoked via NPX.
+const __filename_esm = fileURLToPath(import.meta.url);
+const ROOT = path.join(path.dirname(__filename_esm), "app");
 const SERVER_ROOT = path.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
 const API_PREFIX = "/api/v0";
 const isDevelopment = process.env["NODE_ENV"] === "development";
 
-const app = express();
+export const app = express();
 app.use(loggerMiddleware);
 app.use(httpMetricsMiddleware);
 const projectStore = new ProjectStore(SERVER_ROOT);
+const manifestService = new ManifestService(projectStore);
 const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
-const packageController = new PackageController(projectStore);
+const packageController = new PackageController(projectStore, manifestService);
 const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const compileController = new CompileController(projectStore);
+const materializationService = new MaterializationService(
+   projectStore,
+   manifestService,
+);
+const materializationController = new MaterializationController(
+   materializationService,
+);
+const manifestController = new ManifestController(
+   projectStore,
+   manifestService,
+);
 
 export const mcpApp = express();
 
@@ -262,7 +278,9 @@ app.use(
       credentials: true,
    }),
 );
-app.use(bodyParser.json());
+
+// Set body-parser JSON limit to 1Mb (default: 100kb)
+app.use(bodyParser.json({ limit: "1mb" }));
 
 // Register health check endpoints on main app:
 // - Required for production/Kubernetes monitoring (main server on PUBLISHER_PORT)
@@ -579,7 +597,7 @@ app.post(
                req.params.projectName,
                req.params.connectionName,
                req.body.sqlStatement as string,
-               req.query.options as string,
+               req.body.options as string,
             ),
          );
       } catch (error) {
@@ -650,9 +668,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
+      const autoLoadManifest = req.query.autoLoadManifest === "true";
       const _package = await packageController.addPackage(
          req.params.projectName,
          req.body,
+         { autoLoadManifest },
       );
       res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
@@ -817,12 +837,29 @@ app.get(
          // Express stores wildcard matches in params['0']
          const notebookPath = (req.params as Record<string, string>)["0"];
 
+         // Parse optional filter_params (JSON query string) and bypass_filters
+         let filterParams: Record<string, string | string[]> | undefined;
+         if (typeof req.query.filter_params === "string") {
+            try {
+               filterParams = JSON.parse(req.query.filter_params);
+            } catch {
+               res.status(400).json({
+                  error: "Invalid filter_params: must be valid JSON",
+               });
+               return;
+            }
+         }
+         const bypassFilters =
+            req.query.bypass_filters === "true" ? true : undefined;
+
          res.status(200).json(
             await modelController.executeNotebookCell(
                req.params.projectName,
                req.params.packageName,
                notebookPath,
                cellIndex,
+               filterParams,
+               bypassFilters,
             ),
          );
       } catch (error) {
@@ -879,6 +916,10 @@ app.post(
                req.body.queryName as string,
                req.body.query as string,
                req.body.compactJson === true,
+               (req.body.filterParams ?? req.body.sourceFilters) as
+                  | Record<string, string | string[]>
+                  | undefined,
+               req.body.bypassFilters === true ? true : undefined,
             ),
          );
       } catch (error) {
@@ -932,6 +973,173 @@ app.post(
    },
 );
 
+// ==================== MATERIALIZATION ROUTES ====================
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.createMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(201).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const limit = req.query.limit
+            ? parseInt(req.query.limit as string, 10)
+            : undefined;
+         const offset = req.query.offset
+            ? parseInt(req.query.offset as string, 10)
+            : undefined;
+         const builds = await materializationController.listMaterializations(
+            req.params.projectName,
+            req.params.packageName,
+            { limit, offset },
+         );
+         res.status(200).json(builds);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.getMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(200).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`,
+   async (req, res) => {
+      try {
+         const result = await materializationController.teardownPackage(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(200).json(result);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "start") {
+            const build = await materializationController.startMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(202).json(build);
+         } else if (action === "stop") {
+            const build = await materializationController.stopMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(200).json(build);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`,
+            );
+         }
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.delete(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         await materializationController.deleteMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(204).send();
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+// ==================== MANIFEST ROUTES ====================
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const manifest = await manifestController.getManifest(
+            req.params.projectName,
+            req.params.packageName,
+         );
+         res.status(200).json(manifest);
+      } catch (error) {
+         logger.error("Get manifest error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "reload") {
+            const manifest = await manifestController.reloadManifest(
+               req.params.projectName,
+               req.params.packageName,
+            );
+            res.status(200).json(manifest);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'reload'.`,
+            );
+         }
+      } catch (error) {
+         logger.error("Manifest action error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));

package/src/service/connection.ts

@@ -1093,10 +1093,7 @@ export async function createProjectConnections(
                },
                poolOptions: {
                   min: 1,
-                  max: 5,
-                  testOnBorrow: false,
-                  testOnReturn: false,
-                  testWhileIdle: true,
+                  max: 20,
                },
             };
             const snowflakeConnection = new SnowflakeConnection(