@malloy-publisher/server 0.0.187 → 0.0.189-dev

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.187",
+   "version": "0.0.189-dev",
    "main": "dist/server.js",
    "bin": {
       "malloy-publisher": "dist/server.js"
@@ -14,7 +14,9 @@
       "access": "public"
    },
    "scripts": {
-      "test": "bun test --timeout 100000",
+      "test": "bun run test:unit && bun run test:integration",
+      "test:unit": "bun test --timeout 100000 src",
+      "test:integration": "bun test --timeout 100000 tests",
       "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
       "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
       "start": "NODE_ENV=production node ./dist/server.js",
@@ -56,7 +58,7 @@
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
-      "duckdb": "1.4.3",
+      "duckdb": "1.4.4",
       "express": "^4.21.0",
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",

package/src/controller/connection.controller.ts

@@ -141,7 +141,10 @@ export class ConnectionController {
          if (!source) {
             throw new ConnectionError(`Table ${tablePath} not found`);
          }
-
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof source === "string") {
+            throw new ConnectionError(source);
+         }
          return {
             source: JSON.stringify(source),
             resource: tablePath,
@@ -173,11 +176,11 @@ export class ConnectionController {
       if (!projectName || !connectionName) {
          throw new BadRequestError("Connection payload is required");
       }
-      const { dbConnection } = await this.connectionService.getConnection(
-         projectName,
-         connectionName,
-      );
-      return dbConnection;
+      // Prefer the in-memory API connection (which was materialized by the
+      // project on load and carries `attributes`). The DB row stores the
+      // raw config and FK columns, which aren't the ApiConnection shape.
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.getApiConnection(connectionName);
    }
 
    public async listConnections(projectName: string): Promise<ApiConnection[]> {
@@ -231,22 +234,26 @@ export class ConnectionController {
          projectName,
          connectionName,
       );
-
       try {
+         const schema = await (
+            malloyConnection as Connection & {
+               fetchSelectSchema: (params: {
+                  connection: string;
+                  selectStr: string;
+               }) => Promise<unknown>;
+            }
+         ).fetchSelectSchema({
+            connection: connectionName,
+            selectStr: sqlStatement,
+         });
+
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof schema === "string") {
+            throw new ConnectionError(schema);
+         }
+
          return {
-            source: JSON.stringify(
-               await (
-                  malloyConnection as Connection & {
-                     fetchSelectSchema: (params: {
-                        connection: string;
-                        selectStr: string;
-                     }) => Promise<unknown>;
-                  }
-               ).fetchSelectSchema({
-                  connection: connectionName,
-                  selectStr: sqlStatement,
-               }),
-            ),
+            source: JSON.stringify(schema),
          };
       } catch (error) {
          throw new ConnectionError((error as Error).message);

package/src/controller/manifest.controller.ts

@@ -0,0 +1,29 @@
+import { ManifestService } from "../service/manifest_service";
+import { ProjectStore } from "../service/project_store";
+import { resolveProjectId } from "../service/resolve_project";
+
+export class ManifestController {
+   constructor(
+      private projectStore: ProjectStore,
+      private manifestService: ManifestService,
+   ) {}
+
+   async getManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      // Verify the package exists so we return 404 instead of an empty manifest.
+      const project = await this.projectStore.getProject(projectName, false);
+      await project.getPackage(packageName, false);
+      return this.manifestService.getManifest(projectId, packageName);
+   }
+
+   async reloadManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      return this.manifestService.reloadManifest(
+         projectId,
+         packageName,
+         projectName,
+      );
+   }
+}

package/src/controller/materialization.controller.ts

@@ -0,0 +1,125 @@
+import { BadRequestError } from "../errors";
+import { MaterializationService } from "../service/materialization_service";
+
+export class MaterializationController {
+   constructor(private materializationService: MaterializationService) {}
+
+   async createMaterialization(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateCreateBody(body);
+      return this.materializationService.createMaterialization(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateCreateBody(body: Record<string, unknown>): {
+      forceRefresh?: boolean;
+      autoLoadManifest?: boolean;
+   } {
+      const result: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {};
+      if (body.forceRefresh !== undefined) {
+         if (typeof body.forceRefresh !== "boolean") {
+            throw new BadRequestError("forceRefresh must be a boolean");
+         }
+         result.forceRefresh = body.forceRefresh;
+      }
+      if (body.autoLoadManifest !== undefined) {
+         if (typeof body.autoLoadManifest !== "boolean") {
+            throw new BadRequestError("autoLoadManifest must be a boolean");
+         }
+         result.autoLoadManifest = body.autoLoadManifest;
+      }
+      return result;
+   }
+
+   async startMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.startMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async stopMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.stopMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async listMaterializations(
+      projectName: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ) {
+      return this.materializationService.listMaterializations(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   async getMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.getMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async deleteMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.deleteMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async teardownPackage(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateTeardownBody(body);
+      return this.materializationService.teardownPackage(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateTeardownBody(body: Record<string, unknown>): {
+      dryRun?: boolean;
+   } {
+      const options: { dryRun?: boolean } = {};
+      if (body.dryRun !== undefined) {
+         if (typeof body.dryRun !== "boolean") {
+            throw new BadRequestError("dryRun must be a boolean");
+         }
+         options.dryRun = body.dryRun;
+      }
+      return options;
+   }
+}

package/src/controller/model.controller.ts

@@ -7,8 +7,6 @@ type ApiNotebook = components["schemas"]["Notebook"];
 type ApiModel = components["schemas"]["Model"];
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
-export type ListModelsFilterEnum =
-   components["parameters"]["ListModelsFilterEnum"];
 export class ModelController {
    private projectStore: ProjectStore;
 

package/src/controller/package.controller.ts

@@ -2,15 +2,19 @@ import * as path from "path";
 import { components } from "../api";
 import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
+import { logger } from "../logger";
+import { ManifestService } from "../service/manifest_service";
 import { ProjectStore } from "../service/project_store";
 
 type ApiPackage = components["schemas"]["Package"];
 
 export class PackageController {
    private projectStore: ProjectStore;
+   private manifestService: ManifestService;
 
-   constructor(projectStore: ProjectStore) {
+   constructor(projectStore: ProjectStore, manifestService: ManifestService) {
       this.projectStore = projectStore;
+      this.manifestService = manifestService;
    }
 
    public async listPackages(projectName: string): Promise<ApiPackage[]> {
@@ -32,7 +36,11 @@ export class PackageController {
       return _package.getPackageMetadata();
    }
 
-   async addPackage(projectName: string, body: ApiPackage) {
+   async addPackage(
+      projectName: string,
+      body: ApiPackage,
+      options?: { autoLoadManifest?: boolean },
+   ) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -46,9 +54,52 @@ export class PackageController {
       const result = await project.addPackage(body.name);
       await this.projectStore.addPackageToDatabase(projectName, body.name);
 
+      if (options?.autoLoadManifest === true) {
+         await this.tryLoadExistingManifest(projectName, body.name);
+      }
+
       return result;
    }
 
+   /**
+    * If there are already manifest entries for this package (e.g. from a
+    * previous materialization run), reload all models with the manifest so
+    * persist references resolve to the materialized tables immediately.
+    */
+   private async tryLoadExistingManifest(
+      projectName: string,
+      packageName: string,
+   ): Promise<void> {
+      try {
+         const repository = this.projectStore.storageManager.getRepository();
+         const dbProject = await repository.getProjectByName(projectName);
+         if (!dbProject) return;
+
+         const manifest = await this.manifestService.getManifest(
+            dbProject.id,
+            packageName,
+         );
+         if (Object.keys(manifest.entries).length === 0) return;
+
+         await this.manifestService.reloadManifest(
+            dbProject.id,
+            packageName,
+            projectName,
+         );
+         logger.info("Auto-loaded existing manifest for added package", {
+            projectName,
+            packageName,
+            entryCount: Object.keys(manifest.entries).length,
+         });
+      } catch (error) {
+         logger.warn("Failed to auto-load manifest for package", {
+            projectName,
+            packageName,
+            error,
+         });
+      }
+   }
+
    public async deletePackage(projectName: string, packageName: string) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();

package/src/errors.ts

@@ -20,6 +20,12 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
       return httpError(502, error.message);
+   } else if (error instanceof MaterializationNotFoundError) {
+      return httpError(404, error.message);
+   } else if (error instanceof MaterializationConflictError) {
+      return httpError(409, error.message);
+   } else if (error instanceof InvalidStateTransitionError) {
+      return httpError(409, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -90,3 +96,21 @@ export class FrozenConfigError extends Error {
       super(message);
    }
 }
+
+export class MaterializationNotFoundError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class MaterializationConflictError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class InvalidStateTransitionError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/server.ts

@@ -20,7 +20,11 @@ import { ModelController } from "./controller/model.controller";
 import { PackageController } from "./controller/package.controller";
 import { QueryController } from "./controller/query.controller";
 import { WatchModeController } from "./controller/watch-mode.controller";
-import { internalErrorToHttpError, NotImplementedError } from "./errors";
+import {
+   BadRequestError,
+   internalErrorToHttpError,
+   NotImplementedError,
+} from "./errors";
 import {
    drainingGuard,
    registerHealthEndpoints,
@@ -28,7 +32,11 @@ import {
 } from "./health";
 import { logger, loggerMiddleware } from "./logger";
 
+import { ManifestController } from "./controller/manifest.controller";
+import { MaterializationController } from "./controller/materialization.controller";
 import { initializeMcpServer } from "./mcp/server";
+import { ManifestService } from "./service/manifest_service";
+import { MaterializationService } from "./service/materialization_service";
 import { ProjectStore } from "./service/project_store";
 
 /** Normalize an Express query param into a string[] or undefined. */
@@ -125,17 +133,29 @@ const SERVER_ROOT = path.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
 const API_PREFIX = "/api/v0";
 const isDevelopment = process.env["NODE_ENV"] === "development";
 
-const app = express();
+export const app = express();
 app.use(loggerMiddleware);
 app.use(httpMetricsMiddleware);
 const projectStore = new ProjectStore(SERVER_ROOT);
+const manifestService = new ManifestService(projectStore);
 const watchModeController = new WatchModeController(projectStore);
 const connectionController = new ConnectionController(projectStore);
 const modelController = new ModelController(projectStore);
-const packageController = new PackageController(projectStore);
+const packageController = new PackageController(projectStore, manifestService);
 const databaseController = new DatabaseController(projectStore);
 const queryController = new QueryController(projectStore);
 const compileController = new CompileController(projectStore);
+const materializationService = new MaterializationService(
+   projectStore,
+   manifestService,
+);
+const materializationController = new MaterializationController(
+   materializationService,
+);
+const manifestController = new ManifestController(
+   projectStore,
+   manifestService,
+);
 
 export const mcpApp = express();
 
@@ -262,7 +282,9 @@ app.use(
       credentials: true,
    }),
 );
-app.use(bodyParser.json());
+
+// Set body-parser JSON limit to 1Mb (default: 100kb)
+app.use(bodyParser.json({ limit: "1mb" }));
 
 // Register health check endpoints on main app:
 // - Required for production/Kubernetes monitoring (main server on PUBLISHER_PORT)
@@ -579,7 +601,7 @@ app.post(
                req.params.projectName,
                req.params.connectionName,
                req.body.sqlStatement as string,
-               req.query.options as string,
+               req.body.options as string,
             ),
          );
       } catch (error) {
@@ -650,9 +672,11 @@ app.get(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
 
 app.post(`${API_PREFIX}/projects/:projectName/packages`, async (req, res) => {
    try {
+      const autoLoadManifest = req.query.autoLoadManifest === "true";
       const _package = await packageController.addPackage(
          req.params.projectName,
          req.body,
+         { autoLoadManifest },
       );
       res.status(200).json(_package?.getPackageMetadata());
    } catch (error) {
@@ -953,6 +977,173 @@ app.post(
    },
 );
 
+// ==================== MATERIALIZATION ROUTES ====================
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.createMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(201).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations`,
+   async (req, res) => {
+      try {
+         const limit = req.query.limit
+            ? parseInt(req.query.limit as string, 10)
+            : undefined;
+         const offset = req.query.offset
+            ? parseInt(req.query.offset as string, 10)
+            : undefined;
+         const builds = await materializationController.listMaterializations(
+            req.params.projectName,
+            req.params.packageName,
+            { limit, offset },
+         );
+         res.status(200).json(builds);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const build = await materializationController.getMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(200).json(build);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/teardown`,
+   async (req, res) => {
+      try {
+         const result = await materializationController.teardownPackage(
+            req.params.projectName,
+            req.params.packageName,
+            req.body || {},
+         );
+         res.status(200).json(result);
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "start") {
+            const build = await materializationController.startMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(202).json(build);
+         } else if (action === "stop") {
+            const build = await materializationController.stopMaterialization(
+               req.params.projectName,
+               req.params.packageName,
+               req.params.materializationId,
+            );
+            res.status(200).json(build);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'start' or 'stop'.`,
+            );
+         }
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.delete(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`,
+   async (req, res) => {
+      try {
+         await materializationController.deleteMaterialization(
+            req.params.projectName,
+            req.params.packageName,
+            req.params.materializationId,
+         );
+         res.status(204).send();
+      } catch (error) {
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+// ==================== MANIFEST ROUTES ====================
+
+app.get(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const manifest = await manifestController.getManifest(
+            req.params.projectName,
+            req.params.packageName,
+         );
+         res.status(200).json(manifest);
+      } catch (error) {
+         logger.error("Get manifest error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
+app.post(
+   `${API_PREFIX}/projects/:projectName/packages/:packageName/manifest`,
+   async (req, res) => {
+      try {
+         const action = req.query.action;
+         if (action === "reload") {
+            const manifest = await manifestController.reloadManifest(
+               req.params.projectName,
+               req.params.packageName,
+            );
+            res.status(200).json(manifest);
+         } else {
+            throw new BadRequestError(
+               `Unsupported action '${String(action ?? "")}'. Expected 'reload'.`,
+            );
+         }
+      } catch (error) {
+         logger.error("Manifest action error", { error });
+         const { json, status } = internalErrorToHttpError(error as Error);
+         res.status(status).json(json);
+      }
+   },
+);
+
 // Modify the catch-all route to only serve index.html in production
 if (!isDevelopment) {
    app.get("*", (_req, res) => res.sendFile(path.resolve(ROOT, "index.html")));