@malloy-publisher/server 0.0.182 → 0.0.183-dev

package/src/server.ts

@@ -6,13 +6,14 @@ import {
 } from "./instrumentation";
 
 import { StreamableHTTPServerTransport } from "@modelcontextprotocol/sdk/server/streamableHttp.js";
-import * as bodyParser from "body-parser";
+import bodyParser from "body-parser";
 import cors from "cors";
 import express from "express";
 import * as http from "http";
 import { createProxyMiddleware } from "http-proxy-middleware";
 import { AddressInfo } from "net";
 import * as path from "path";
+import { fileURLToPath } from "url";
 import { CompileController } from "./controller/compile.controller";
 import { ConnectionController } from "./controller/connection.controller";
 import { DatabaseController } from "./controller/database.controller";
@@ -20,7 +21,11 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { WatchModeController } from "./controller/watch-mode.controller";
-import { internalErrorToHttpError, NotImplementedError } from "./errors";
+import {
+   BadRequestError,
+   internalErrorToHttpError,
+   NotImplementedError,
+} from "./errors";
 import {
    drainingGuard,
    registerHealthEndpoints,
@@ -28,7 +33,11 @@ import {
 } from "./health";
 import { logger, loggerMiddleware } from "./logger";
 
+import { ManifestController } from "./controller/manifest.controller";
+import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { ManifestService } from "./service/manifest_service";
+import { MaterializationService } from "./service/materialization_service";
 import { ProjectStore } from "./service/project_store";
 
 /** Normalize an Express query param into a string[] or undefined. */
@@ -112,30 +121,37 @@ const SHUTDOWN_DRAIN_DURATION_SECONDS = Number(
 const SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(
    process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0,
 );
-// Find the app directory - handle NPX vs local execution
-let ROOT: string;
-if (require.main) {
-   // Use the main module's directory (works for NPX and direct execution)
-   ROOT = path.join(path.dirname(require.main.filename), "app");
-} else {
-   // Fallback to current script directory
-   ROOT = path.join(path.dirname(process.argv[1] || __filename), "app");
-}
+// Find the app directory relative to this bundled server file.
+// Works under both ESM (import.meta.url) and when invoked via NPX.
+const __filename_esm = fileURLToPath(import.meta.url);
+const ROOT = path.join(path.dirname(__filename_esm), "app");
 const SERVER_ROOT = path.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
 const API_PREFIX = "/api/v0";
 const isDevelopment = process.env["NODE_ENV"] === "development";
 
-const app = express();
+export const app = express();
 app.use(loggerMiddleware);
 app.use(httpMetricsMiddleware);
 const projectStore = new ProjectStore(SERVER_ROOT);
+const manifestService = new ManifestService(projectStore);
 const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
-const packageController = new PackageController(projectStore);
+const packageController = new PackageController(projectStore, manifestService);
 const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const compileController = new CompileController(projectStore);
+const materializationService = new MaterializationService(
+   projectStore,
+   manifestService,
+);
+const materializationController = new MaterializationController(
+   materializationService,
+);
+const manifestController = new ManifestController(
+   projectStore,
+   manifestService,
+);
 
 export const mcpApp = express();
 
@@ -262,7 +278,9 @@ app.use(
       credentials: true,
    }),
 );
-app.use(bodyParser.json());
+
+// Set body-parser JSON limit to 1Mb (default: 100kb)
+app.use(bodyParser.json({ limit: "1mb" }));
 
 // Register health check endpoints on main app:
 // - Required for production/Kubernetes monitoring (main server on PUBLISHER_PORT)
@@ -579,7 +597,7 @@ app.post(
                req.params.projectName,
                req.params.connectionName,
                req.body.sqlStatement as string,
-               req.query.options as string,
+               req.body.options as string,
             ),
          );
       } catch (error) {
@@ -650,9 +668,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
+      const autoLoadManifest = req.query.autoLoadManifest === "true";
       const _package = await packageController.addPackage(
          req.params.projectName,
          req.body,
+         { autoLoadManifest },
       );
       res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
@@ -953,6 +973,173 @@ app.post(
    },
 );
 
+// ==================== MATERIALIZATION ROUTES ====================
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.createMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(201).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const limit = req.query.limit
+            ? parseInt(req.query.limit as string, 10)
+            : undefined;
+         const offset = req.query.offset
+            ? parseInt(req.query.offset as string, 10)
+            : undefined;
+         const builds = await materializationController.listMaterializations(
+            req.params.projectName,
+            req.params.packageName,
+            { limit, offset },
+         );
+         res.status(200).json(builds);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.getMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(200).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`,
+   async (req, res) => {
+      try {
+         const result = await materializationController.teardownPackage(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(200).json(result);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "start") {
+            const build = await materializationController.startMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(202).json(build);
+         } else if (action === "stop") {
+            const build = await materializationController.stopMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(200).json(build);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`,
+            );
+         }
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.delete(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         await materializationController.deleteMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(204).send();
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+// ==================== MANIFEST ROUTES ====================
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const manifest = await manifestController.getManifest(
+            req.params.projectName,
+            req.params.packageName,
+         );
+         res.status(200).json(manifest);
+      } catch (error) {
+         logger.error("Get manifest error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "reload") {
+            const manifest = await manifestController.reloadManifest(
+               req.params.projectName,
+               req.params.packageName,
+            );
+            res.status(200).json(manifest);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'reload'.`,
+            );
+         }
+      } catch (error) {
+         logger.error("Manifest action error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));

package/src/service/connection.ts

@@ -1093,10 +1093,7 @@ export async function createProjectConnections(
                },
                poolOptions: {
                   min: 1,
-                  max: 5,
-                  testOnBorrow: false,
-                  testOnReturn: false,
-                  testWhileIdle: true,
+                  max: 20,
                },
             };
             const snowflakeConnection = new SnowflakeConnection(

package/src/service/filter.spec.ts

@@ -170,6 +170,61 @@ describe("service/filter", () => {
          const filters = parseFilters(["#(doc) some docs", "# hidden"]);
          expect(filters).toHaveLength(0);
       });
+
+      it("deduplicates by name, later annotations win (extend pattern)", () => {
+         const annotations = [
+            // Base source annotations (come first in blockNotes via inherits chain)
+            "#(filter) name=Manufacturer dimension=Manufacturer type=in",
+            "#(filter) name=Subject dimension=Subject type=like",
+            '#(filter) name="Major Recall" dimension="Major Recall" type=equal',
+            // Extending source annotations (come later, should win)
+            "#(filter) name=Manufacturer dimension=Manufacturer type=equal required",
+            "#(filter) name=Subject dimension=Subject type=like",
+         ];
+         const filters = parseFilters(annotations);
+         // 3 unique names: Manufacturer, Subject, Major Recall
+         expect(filters).toHaveLength(3);
+
+         // Manufacturer: child overrides base (in → equal, gains required)
+         const mfr = filters.find((f) => f.name === "Manufacturer");
+         expect(mfr).toBeDefined();
+         expect(mfr!.type).toBe("equal");
+         expect(mfr!.required).toBe(true);
+
+         // Subject: child re-declares identically, no visible change
+         const subj = filters.find((f) => f.name === "Subject");
+         expect(subj).toBeDefined();
+         expect(subj!.type).toBe("like");
+         expect(subj!.required).toBeFalsy();
+
+         // Major Recall: only on base, preserved in child
+         const major = filters.find((f) => f.name === "Major Recall");
+         expect(major).toBeDefined();
+         expect(major!.type).toBe("equal");
+         expect(major!.dimension).toBe("Major Recall");
+      });
+
+      it("child can remove required flag by overriding", () => {
+         const annotations = [
+            "#(filter) name=status dimension=status type=equal required",
+            "#(filter) name=status dimension=status type=equal",
+         ];
+         const filters = parseFilters(annotations);
+         expect(filters).toHaveLength(1);
+         expect(filters[0].name).toBe("status");
+         expect(filters[0].required).toBeFalsy();
+      });
+
+      it("child can change filter type by overriding", () => {
+         const annotations = [
+            "#(filter) name=category dimension=category type=in",
+            "#(filter) name=category dimension=category type=equal required",
+         ];
+         const filters = parseFilters(annotations);
+         expect(filters).toHaveLength(1);
+         expect(filters[0].type).toBe("equal");
+         expect(filters[0].required).toBe(true);
+      });
    });
 
    // -----------------------------------------------------------------------

package/src/service/filter.ts

@@ -127,14 +127,19 @@ export function parseFilterAnnotation(
  * (as found on a Malloy source's `blockNotes`).
  */
 export function parseFilters(annotations: string[]): FilterDefinition[] {
-   const filters: FilterDefinition[] = [];
+   // Use a Map keyed by filter name so that later annotations (from an
+   // extending source) override earlier ones (from the base source).
+   // This is important when `source: child is parent extend {}` inherits
+   // blockNotes from the parent — the child's annotations come last and
+   // should win.
+   const byName = new Map<string, FilterDefinition>();
    for (const annotation of annotations) {
       const parsed = parseFilterAnnotation(annotation);
       if (parsed) {
-         filters.push(parsed);
+         byName.set(parsed.name, parsed);
       }
    }
-   return filters;
+   return [...byName.values()];
 }
 
 // ---------------------------------------------------------------------------

package/src/service/filter_integration.spec.ts

@@ -94,6 +94,45 @@ import "orders_optional.malloy"
 run: orders -> summary
 `;
 
+// Base source with 3 filters: region (in), status (equal), customer_id (equal, required)
+const MODEL_BASE_FOR_EXTEND = `
+#(filter) name=region dimension=region type=in
+#(filter) name=status dimension=status type=equal
+#(filter) name=tenant dimension=customer_id type=equal required
+source: base_orders is duckdb.table('orders') extend {
+   primary_key: order_id
+
+   measure:
+      order_count is count()
+      total_amount is sum(amount)
+
+   view: summary is {
+      aggregate: order_count, total_amount
+   }
+}
+`;
+
+// Extending source: overrides region (in → equal), overrides tenant
+// (removes required), keeps status from base unchanged
+const MODEL_CHILD_EXTEND = `
+import "base_orders.malloy"
+
+#(filter) name=region dimension=region type=equal
+#(filter) name=tenant dimension=customer_id type=equal
+source: child_orders is base_orders extend {}
+`;
+
+// Notebook against the extended source
+const NOTEBOOK_EXTEND = `>>>markdown
+# Extend Test
+
+>>>malloy
+import "child_orders.malloy"
+
+>>>malloy
+run: child_orders -> summary
+`;
+
 beforeAll(async () => {
    await fs.mkdir(TEST_DB_DIR, { recursive: true });
    await fs.mkdir(TEST_PKG_DIR, { recursive: true });
@@ -619,4 +658,168 @@ describe("filter integration", () => {
          expect(markdownCell.text).toContain("Test Notebook");
       });
    });
+
+   // -----------------------------------------------------------------------
+   // Extended source filter inheritance
+   // -----------------------------------------------------------------------
+   describe("extended source filter inheritance", () => {
+      beforeEach(async () => {
+         await writeFile("base_orders.malloy", MODEL_BASE_FOR_EXTEND);
+         await writeFile("child_orders.malloy", MODEL_CHILD_EXTEND);
+         await writeFile("extend_notebook.malloynb", NOTEBOOK_EXTEND);
+      });
+
+      it("inherits base-only filters on extended source", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         const sources = model.getSources();
+         const child = sources!.find((s) => s.name === "child_orders");
+         expect(child).toBeDefined();
+         expect(child!.filters).toBeDefined();
+
+         // status is defined only on the base — it should carry through
+         const statusFilter = child!.filters!.find((f) => f.name === "status");
+         expect(statusFilter).toBeDefined();
+         expect(statusFilter!.type).toBe("equal");
+      });
+
+      it("child overrides base filter type", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         const sources = model.getSources();
+         const child = sources!.find((s) => s.name === "child_orders");
+         expect(child).toBeDefined();
+
+         // region: base=in, child overrides to equal
+         const regionFilter = child!.filters!.find((f) => f.name === "region");
+         expect(regionFilter).toBeDefined();
+         expect(regionFilter!.type).toBe("equal");
+      });
+
+      it("child can remove required flag by overriding", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         const sources = model.getSources();
+         const child = sources!.find((s) => s.name === "child_orders");
+         expect(child).toBeDefined();
+
+         // tenant: base=required, child overrides without required
+         const tenantFilter = child!.filters!.find((f) => f.name === "tenant");
+         expect(tenantFilter).toBeDefined();
+         expect(tenantFilter!.required).toBeFalsy();
+      });
+
+      it("has exactly the expected merged filter set", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         const sources = model.getSources();
+         const child = sources!.find((s) => s.name === "child_orders");
+         expect(child).toBeDefined();
+
+         // 3 unique filter names: region, status (from base), tenant
+         const filterNames = child!.filters!.map((f) => f.name).sort();
+         expect(filterNames).toEqual(["region", "status", "tenant"]);
+      });
+
+      it("applies inherited filter to query on extended source", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         // status=active is inherited from the base; should work on child
+         const { compactResult } = await model.getQueryResults(
+            "child_orders",
+            "summary",
+            undefined,
+            { status: "active" },
+         );
+         const rows = asRows(compactResult);
+         expect(rows.length).toBe(1);
+         // 4 active rows: US(2), EU(1), APAC(1)
+         expect(Number(rows[0].order_count)).toBe(4);
+      });
+
+      it("applies overridden filter to query on extended source", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         // region is overridden to type=equal on the child
+         const { compactResult } = await model.getQueryResults(
+            "child_orders",
+            "summary",
+            undefined,
+            { region: "US" },
+         );
+         const rows = asRows(compactResult);
+         expect(rows.length).toBe(1);
+         // 2 US rows
+         expect(Number(rows[0].order_count)).toBe(2);
+      });
+
+      it("no longer requires base's required filter after child override", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "child_orders.malloy",
+            getConnections(),
+         );
+
+         // On the base, tenant is required. On the child, it's not.
+         // Running without tenant should NOT throw.
+         const { compactResult } = await model.getQueryResults(
+            "child_orders",
+            "summary",
+         );
+         const rows = asRows(compactResult);
+         expect(rows.length).toBe(1);
+         expect(Number(rows[0].order_count)).toBe(6);
+      });
+
+      it("applies inherited filters to notebook cells", async () => {
+         const model = await Model.create(
+            "test-pkg",
+            TEST_PKG_DIR,
+            "extend_notebook.malloynb",
+            getConnections(),
+         );
+
+         // Apply status=cancelled (inherited from base) via notebook cell
+         const codeCell = await model.executeNotebookCell(2, {
+            status: "cancelled",
+         });
+         expect(codeCell.result).toBeDefined();
+
+         const rows = parseNotebookResult(codeCell.result!);
+         expect(rows.length).toBe(1);
+         // 2 cancelled rows: EU(1), APAC(1)
+         expect(Number(rows[0].order_count)).toBe(2);
+      });
+   });
 });