@malloy-publisher/server 0.0.178 → 0.0.180-dev-v1

package/src/server.ts

@@ -20,7 +20,11 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { WatchModeController } from "./controller/watch-mode.controller";
-import { internalErrorToHttpError, NotImplementedError } from "./errors";
+import {
+   BadRequestError,
+   internalErrorToHttpError,
+   NotImplementedError,
+} from "./errors";
 import {
    drainingGuard,
    registerHealthEndpoints,
@@ -28,7 +32,11 @@ import {
 } from "./health";
 import { logger, loggerMiddleware } from "./logger";
 
+import { ManifestController } from "./controller/manifest.controller";
+import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { ManifestService } from "./service/manifest_service";
+import { MaterializationService } from "./service/materialization_service";
 import { ProjectStore } from "./service/project_store";
 
 /** Normalize an Express query param into a string[] or undefined. */
@@ -125,17 +133,29 @@ const SERVER_ROOT = path.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
 const API_PREFIX = "/api/v0";
 const isDevelopment = process.env["NODE_ENV"] === "development";
 
-const app = express();
+export const app = express();
 app.use(loggerMiddleware);
 app.use(httpMetricsMiddleware);
 const projectStore = new ProjectStore(SERVER_ROOT);
+const manifestService = new ManifestService(projectStore);
 const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
-const packageController = new PackageController(projectStore);
+const packageController = new PackageController(projectStore, manifestService);
 const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const compileController = new CompileController(projectStore);
+const materializationService = new MaterializationService(
+   projectStore,
+   manifestService,
+);
+const materializationController = new MaterializationController(
+   materializationService,
+);
+const manifestController = new ManifestController(
+   projectStore,
+   manifestService,
+);
 
 export const mcpApp = express();
 
@@ -262,7 +282,9 @@ app.use(
       credentials: true,
    }),
 );
-app.use(bodyParser.json());
+
+// Set body-parser JSON limit to 1Mb (default: 100kb)
+app.use(bodyParser.json({ limit: "1mb" }));
 
 // Register health check endpoints on main app:
 // - Required for production/Kubernetes monitoring (main server on PUBLISHER_PORT)
@@ -579,7 +601,7 @@ app.post(
                req.params.projectName,
                req.params.connectionName,
                req.body.sqlStatement as string,
-               req.query.options as string,
+               req.body.options as string,
             ),
          );
       } catch (error) {
@@ -650,9 +672,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
+      const autoLoadManifest = req.query.autoLoadManifest === "true";
       const _package = await packageController.addPackage(
          req.params.projectName,
          req.body,
+         { autoLoadManifest },
       );
       res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
@@ -817,12 +841,29 @@ app.get(
          // Express stores wildcard matches in params['0']
          const notebookPath = (req.params as Record<string, string>)["0"];
 
+         // Parse optional filter_params (JSON query string) and bypass_filters
+         let filterParams: Record<string, string | string[]> | undefined;
+         if (typeof req.query.filter_params === "string") {
+            try {
+               filterParams = JSON.parse(req.query.filter_params);
+            } catch {
+               res.status(400).json({
+                  error: "Invalid filter_params: must be valid JSON",
+               });
+               return;
+            }
+         }
+         const bypassFilters =
+            req.query.bypass_filters === "true" ? true : undefined;
+
          res.status(200).json(
             await modelController.executeNotebookCell(
                req.params.projectName,
                req.params.packageName,
                notebookPath,
                cellIndex,
+               filterParams,
+               bypassFilters,
             ),
          );
       } catch (error) {
@@ -879,6 +920,10 @@ app.post(
                req.body.queryName as string,
                req.body.query as string,
                req.body.compactJson === true,
+               (req.body.filterParams ?? req.body.sourceFilters) as
+                  | Record<string, string | string[]>
+                  | undefined,
+               req.body.bypassFilters === true ? true : undefined,
             ),
          );
       } catch (error) {
@@ -932,6 +977,173 @@ app.post(
    },
 );
 
+// ==================== MATERIALIZATION ROUTES ====================
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.createMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(201).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const limit = req.query.limit
+            ? parseInt(req.query.limit as string, 10)
+            : undefined;
+         const offset = req.query.offset
+            ? parseInt(req.query.offset as string, 10)
+            : undefined;
+         const builds = await materializationController.listMaterializations(
+            req.params.projectName,
+            req.params.packageName,
+            { limit, offset },
+         );
+         res.status(200).json(builds);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.getMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(200).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`,
+   async (req, res) => {
+      try {
+         const result = await materializationController.teardownPackage(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(200).json(result);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "start") {
+            const build = await materializationController.startMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(202).json(build);
+         } else if (action === "stop") {
+            const build = await materializationController.stopMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(200).json(build);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`,
+            );
+         }
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.delete(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         await materializationController.deleteMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(204).send();
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+// ==================== MANIFEST ROUTES ====================
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const manifest = await manifestController.getManifest(
+            req.params.projectName,
+            req.params.packageName,
+         );
+         res.status(200).json(manifest);
+      } catch (error) {
+         logger.error("Get manifest error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "reload") {
+            const manifest = await manifestController.reloadManifest(
+               req.params.projectName,
+               req.params.packageName,
+            );
+            res.status(200).json(manifest);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'reload'.`,
+            );
+         }
+      } catch (error) {
+         logger.error("Manifest action error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));

package/src/service/connection.ts

@@ -1093,10 +1093,7 @@ export async function createProjectConnections(
                },
                poolOptions: {
                   min: 1,
-                  max: 5,
-                  testOnBorrow: false,
-                  testOnReturn: false,
-                  testWhileIdle: true,
+                  max: 20,
                },
             };
             const snowflakeConnection = new SnowflakeConnection(

package/src/service/db_utils.spec.ts

@@ -167,10 +167,11 @@ describe("listTablesForSchema", () => {
          },
       };
 
-      it("queries information_schema.columns with correct schema", async () => {
+      it("queries information_schema.columns wrapped in row_to_json", async () => {
          const m = mockConnection(columnRows);
          const tables = await listTablesForSchema(conn, "public", m.conn);
 
+         expect(m.lastSQL).toContain("row_to_json");
          expect(m.lastSQL).toContain("information_schema.columns");
          expect(m.lastSQL).toContain("table_schema = 'public'");
          expect(tables).toHaveLength(2);
@@ -470,7 +471,7 @@ describe("getSchemasForConnection", () => {
          },
       };
 
-      it("queries information_schema.schemata", async () => {
+      it("queries information_schema.schemata wrapped in row_to_json", async () => {
          const rows = [
             { schema_name: "public" },
             { schema_name: "information_schema" },
@@ -480,6 +481,7 @@ describe("getSchemasForConnection", () => {
          const m = mockConnection(rows);
          const schemas = await getSchemasForConnection(conn, m.conn);
 
+         expect(m.lastSQL).toContain("row_to_json");
          expect(m.lastSQL).toContain("information_schema.schemata");
          expect(schemas).toHaveLength(4);
          expect(schemas.find((s) => s.name === "public")?.isDefault).toBe(true);

package/src/service/db_utils.ts

@@ -175,8 +175,10 @@ async function getSchemasForPostgres(
       throw new Error("Postgres connection is required");
    }
    try {
+      // Wrap in row_to_json because the Malloy Postgres driver's runSQL
+      // de-JSONs each row via row.row (matching Malloy-generated queries).
       const result = await malloyConnection.runSQL(
-         "SELECT schema_name FROM information_schema.schemata ORDER BY schema_name",
+         "SELECT row_to_json(t) as row FROM (SELECT schema_name FROM information_schema.schemata ORDER BY schema_name) t",
       );
       const rows = standardizeRunSQLResult(result);
       return rows.map((row: unknown) => {
@@ -946,8 +948,10 @@ async function listTablesForPostgres(
       throw new Error("Postgres connection is required");
    }
    try {
+      // Wrap in row_to_json because the Malloy Postgres driver's runSQL
+      // de-JSONs each row via row.row (matching Malloy-generated queries).
       const result = await malloyConnection.runSQL(
-         `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${schemaName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`,
+         `SELECT row_to_json(t) as row FROM (SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${schemaName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position) t`,
       );
       const rows = standardizeRunSQLResult(result);
       return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);