@malloy-publisher/server 0.0.182 → 0.0.183-dev

package/package.json

@@ -1,10 +1,10 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.182",
-   "main": "dist/server.js",
+   "version": "0.0.183-dev",
+   "main": "dist/server.mjs",
    "bin": {
-      "malloy-publisher": "dist/server.js"
+      "malloy-publisher": "dist/server.mjs"
    },
    "repository": {
       "type": "git",
@@ -14,14 +14,16 @@
       "access": "public"
    },
    "scripts": {
-      "test": "bun test --timeout 100000",
+      "test": "bun run test:unit && bun run test:integration",
+      "test:unit": "bun test --timeout 100000 src",
+      "test:integration": "bun test --timeout 100000 tests",
       "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
       "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
-      "start": "NODE_ENV=production node ./dist/server.js",
-      "start:init": "NODE_ENV=production node ./dist/server.js --init",
+      "start": "NODE_ENV=production bun run ./dist/server.mjs",
+      "start:init": "NODE_ENV=production bun run ./dist/server.mjs --init",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
       "start:dev:init": "NODE_ENV=development bun --watch src/server.ts --init",
-      "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
+      "start:instrumented": "bun --preload ./dist/instrumentation.mjs ./dist/server.mjs",
       "lint": "bunx eslint \"./src/**/*.ts\"",
       "format": "bunx prettier --write '**/*.{ts,tsx}'",
       "build:app": "cd ../app && NODE_ENV=production bunx vite build",
@@ -32,15 +34,15 @@
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.374",
-      "@malloydata/db-duckdb": "^0.0.374",
-      "@malloydata/db-mysql": "^0.0.374",
-      "@malloydata/db-postgres": "^0.0.374",
-      "@malloydata/db-snowflake": "^0.0.374",
-      "@malloydata/db-trino": "^0.0.374",
-      "@malloydata/malloy": "^0.0.374",
-      "@malloydata/malloy-sql": "^0.0.374",
-      "@malloydata/render-validator": "^0.0.374",
+      "@malloydata/db-bigquery": "^0.0.377",
+      "@malloydata/db-duckdb": "^0.0.377",
+      "@malloydata/db-mysql": "^0.0.377",
+      "@malloydata/db-postgres": "^0.0.377",
+      "@malloydata/db-snowflake": "^0.0.377",
+      "@malloydata/db-trino": "^0.0.377",
+      "@malloydata/malloy": "^0.0.377",
+      "@malloydata/malloy-sql": "^0.0.377",
+      "@malloydata/render-validator": "^0.0.377",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",
@@ -56,7 +58,7 @@
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
-      "duckdb": "1.4.3",
+      "duckdb": "1.4.4",
       "express": "^4.21.0",
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",

package/src/controller/connection.controller.ts

@@ -141,7 +141,10 @@ export class ConnectionController {
          if (!source) {
             throw new ConnectionError(`Table ${tablePath} not found`);
          }
-
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof source === "string") {
+            throw new ConnectionError(source);
+         }
          return {
             source: JSON.stringify(source),
             resource: tablePath,
@@ -173,11 +176,11 @@ export class ConnectionController {
       if (!projectName || !connectionName) {
          throw new BadRequestError("Connection payload is required");
       }
-      const { dbConnection } = await this.connectionService.getConnection(
-         projectName,
-         connectionName,
-      );
-      return dbConnection;
+      // Prefer the in-memory API connection (which was materialized by the
+      // project on load and carries `attributes`). The DB row stores the
+      // raw config and FK columns, which aren't the ApiConnection shape.
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.getApiConnection(connectionName);
    }
 
    public async listConnections(projectName: string): Promise<ApiConnection[]> {
@@ -231,22 +234,26 @@ export class ConnectionController {
          projectName,
          connectionName,
       );
-
       try {
+         const schema = await (
+            malloyConnection as Connection & {
+               fetchSelectSchema: (params: {
+                  connection: string;
+                  selectStr: string;
+               }) => Promise<unknown>;
+            }
+         ).fetchSelectSchema({
+            connection: connectionName,
+            selectStr: sqlStatement,
+         });
+
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof schema === "string") {
+            throw new ConnectionError(schema);
+         }
+
          return {
-            source: JSON.stringify(
-               await (
-                  malloyConnection as Connection & {
-                     fetchSelectSchema: (params: {
-                        connection: string;
-                        selectStr: string;
-                     }) => Promise<unknown>;
-                  }
-               ).fetchSelectSchema({
-                  connection: connectionName,
-                  selectStr: sqlStatement,
-               }),
-            ),
+            source: JSON.stringify(schema),
          };
       } catch (error) {
          throw new ConnectionError((error as Error).message);

package/src/controller/manifest.controller.ts

@@ -0,0 +1,29 @@
+import { ManifestService } from "../service/manifest_service";
+import { ProjectStore } from "../service/project_store";
+import { resolveProjectId } from "../service/resolve_project";
+
+export class ManifestController {
+   constructor(
+      private projectStore: ProjectStore,
+      private manifestService: ManifestService,
+   ) {}
+
+   async getManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      // Verify the package exists so we return 404 instead of an empty manifest.
+      const project = await this.projectStore.getProject(projectName, false);
+      await project.getPackage(packageName, false);
+      return this.manifestService.getManifest(projectId, packageName);
+   }
+
+   async reloadManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      return this.manifestService.reloadManifest(
+         projectId,
+         packageName,
+         projectName,
+      );
+   }
+}

package/src/controller/materialization.controller.ts

@@ -0,0 +1,125 @@
+import { BadRequestError } from "../errors";
+import { MaterializationService } from "../service/materialization_service";
+
+export class MaterializationController {
+   constructor(private materializationService: MaterializationService) {}
+
+   async createMaterialization(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateCreateBody(body);
+      return this.materializationService.createMaterialization(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateCreateBody(body: Record<string, unknown>): {
+      forceRefresh?: boolean;
+      autoLoadManifest?: boolean;
+   } {
+      const result: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {};
+      if (body.forceRefresh !== undefined) {
+         if (typeof body.forceRefresh !== "boolean") {
+            throw new BadRequestError("forceRefresh must be a boolean");
+         }
+         result.forceRefresh = body.forceRefresh;
+      }
+      if (body.autoLoadManifest !== undefined) {
+         if (typeof body.autoLoadManifest !== "boolean") {
+            throw new BadRequestError("autoLoadManifest must be a boolean");
+         }
+         result.autoLoadManifest = body.autoLoadManifest;
+      }
+      return result;
+   }
+
+   async startMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.startMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async stopMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.stopMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async listMaterializations(
+      projectName: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ) {
+      return this.materializationService.listMaterializations(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   async getMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.getMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async deleteMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.deleteMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async teardownPackage(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateTeardownBody(body);
+      return this.materializationService.teardownPackage(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateTeardownBody(body: Record<string, unknown>): {
+      dryRun?: boolean;
+   } {
+      const options: { dryRun?: boolean } = {};
+      if (body.dryRun !== undefined) {
+         if (typeof body.dryRun !== "boolean") {
+            throw new BadRequestError("dryRun must be a boolean");
+         }
+         options.dryRun = body.dryRun;
+      }
+      return options;
+   }
+}

package/src/controller/model.controller.ts

@@ -7,8 +7,6 @@ type ApiNotebook = components["schemas"]["Notebook"];
 type ApiModel = components["schemas"]["Model"];
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
-export type ListModelsFilterEnum =
-   components["parameters"]["ListModelsFilterEnum"];
 export class ModelController {
    private projectStore: ProjectStore;
 

package/src/controller/package.controller.ts

@@ -2,15 +2,19 @@ import * as path from "path";
 import { components } from "../api";
 import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
+import { logger } from "../logger";
+import { ManifestService } from "../service/manifest_service";
 import { ProjectStore } from "../service/project_store";
 
 type ApiPackage = components["schemas"]["Package"];
 
 export class PackageController {
    private projectStore: ProjectStore;
+   private manifestService: ManifestService;
 
-   constructor(projectStore: ProjectStore) {
+   constructor(projectStore: ProjectStore, manifestService: ManifestService) {
       this.projectStore = projectStore;
+      this.manifestService = manifestService;
    }
 
    public async listPackages(projectName: string): Promise<ApiPackage[]> {
@@ -32,7 +36,11 @@ export class PackageController {
       return _package.getPackageMetadata();
    }
 
-   async addPackage(projectName: string, body: ApiPackage) {
+   async addPackage(
+      projectName: string,
+      body: ApiPackage,
+      options?: { autoLoadManifest?: boolean },
+   ) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -46,9 +54,52 @@ export class PackageController {
       const result = await project.addPackage(body.name);
       await this.projectStore.addPackageToDatabase(projectName, body.name);
 
+      if (options?.autoLoadManifest === true) {
+         await this.tryLoadExistingManifest(projectName, body.name);
+      }
+
       return result;
    }
 
+   /**
+    * If there are already manifest entries for this package (e.g. from a
+    * previous materialization run), reload all models with the manifest so
+    * persist references resolve to the materialized tables immediately.
+    */
+   private async tryLoadExistingManifest(
+      projectName: string,
+      packageName: string,
+   ): Promise<void> {
+      try {
+         const repository = this.projectStore.storageManager.getRepository();
+         const dbProject = await repository.getProjectByName(projectName);
+         if (!dbProject) return;
+
+         const manifest = await this.manifestService.getManifest(
+            dbProject.id,
+            packageName,
+         );
+         if (Object.keys(manifest.entries).length === 0) return;
+
+         await this.manifestService.reloadManifest(
+            dbProject.id,
+            packageName,
+            projectName,
+         );
+         logger.info("Auto-loaded existing manifest for added package", {
+            projectName,
+            packageName,
+            entryCount: Object.keys(manifest.entries).length,
+         });
+      } catch (error) {
+         logger.warn("Failed to auto-load manifest for package", {
+            projectName,
+            packageName,
+            error,
+         });
+      }
+   }
+
    public async deletePackage(projectName: string, packageName: string) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();

package/src/errors.ts

@@ -20,6 +20,12 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
       return httpError(502, error.message);
+   } else if (error instanceof MaterializationNotFoundError) {
+      return httpError(404, error.message);
+   } else if (error instanceof MaterializationConflictError) {
+      return httpError(409, error.message);
+   } else if (error instanceof InvalidStateTransitionError) {
+      return httpError(409, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -90,3 +96,21 @@ export class FrozenConfigError extends Error {
       super(message);
    }
 }
+
+export class MaterializationNotFoundError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class MaterializationConflictError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class InvalidStateTransitionError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/mcp/prompts/handlers.ts

@@ -1,4 +1,4 @@
-import * as Handlebars from "handlebars";
+import Handlebars from "handlebars";
 import { z } from "zod";
 import {
    PromptMessageSchema,