@malloy-publisher/server 0.0.181 → 0.0.183-dev

package/package.json

@@ -1,10 +1,10 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.181",
-   "main": "dist/server.js",
+   "version": "0.0.183-dev",
+   "main": "dist/server.mjs",
    "bin": {
-      "malloy-publisher": "dist/server.js"
+      "malloy-publisher": "dist/server.mjs"
    },
    "repository": {
       "type": "git",
@@ -14,14 +14,16 @@
       "access": "public"
    },
    "scripts": {
-      "test": "bun test --timeout 100000",
+      "test": "bun run test:unit && bun run test:integration",
+      "test:unit": "bun test --timeout 100000 src",
+      "test:integration": "bun test --timeout 100000 tests",
       "build": "bun generate-api-types && bun build:app && NODE_ENV=production bun run build.ts",
       "build:server-only": "bun generate-api-types && NODE_ENV=production bun run build.ts",
-      "start": "NODE_ENV=production node ./dist/server.js",
-      "start:init": "NODE_ENV=production node ./dist/server.js --init",
+      "start": "NODE_ENV=production bun run ./dist/server.mjs",
+      "start:init": "NODE_ENV=production bun run ./dist/server.mjs --init",
       "start:dev": "NODE_ENV=development bun --watch src/server.ts",
       "start:dev:init": "NODE_ENV=development bun --watch src/server.ts --init",
-      "start:instrumented": "node --require ./dist/instrumentation.js ./dist/server.js",
+      "start:instrumented": "bun --preload ./dist/instrumentation.mjs ./dist/server.mjs",
       "lint": "bunx eslint \"./src/**/*.ts\"",
       "format": "bunx prettier --write '**/*.{ts,tsx}'",
       "build:app": "cd ../app && NODE_ENV=production bunx vite build",
@@ -32,15 +34,15 @@
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.374",
-      "@malloydata/db-duckdb": "^0.0.374",
-      "@malloydata/db-mysql": "^0.0.374",
-      "@malloydata/db-postgres": "^0.0.374",
-      "@malloydata/db-snowflake": "^0.0.374",
-      "@malloydata/db-trino": "^0.0.374",
-      "@malloydata/malloy": "^0.0.374",
-      "@malloydata/malloy-sql": "^0.0.374",
-      "@malloydata/render-validator": "^0.0.374",
+      "@malloydata/db-bigquery": "^0.0.377",
+      "@malloydata/db-duckdb": "^0.0.377",
+      "@malloydata/db-mysql": "^0.0.377",
+      "@malloydata/db-postgres": "^0.0.377",
+      "@malloydata/db-snowflake": "^0.0.377",
+      "@malloydata/db-trino": "^0.0.377",
+      "@malloydata/malloy": "^0.0.377",
+      "@malloydata/malloy-sql": "^0.0.377",
+      "@malloydata/render-validator": "^0.0.377",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",
@@ -56,7 +58,7 @@
       "class-transformer": "^0.5.1",
       "class-validator": "^0.14.1",
       "cors": "^2.8.5",
-      "duckdb": "1.4.3",
+      "duckdb": "1.4.4",
       "express": "^4.21.0",
       "globals": "^15.9.0",
       "handlebars": "^4.7.8",

package/src/controller/connection.controller.ts

@@ -141,7 +141,10 @@ export class ConnectionController {
          if (!source) {
             throw new ConnectionError(`Table ${tablePath} not found`);
          }
-
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof source === "string") {
+            throw new ConnectionError(source);
+         }
          return {
             source: JSON.stringify(source),
             resource: tablePath,
@@ -173,11 +176,11 @@ export class ConnectionController {
       if (!projectName || !connectionName) {
          throw new BadRequestError("Connection payload is required");
       }
-      const { dbConnection } = await this.connectionService.getConnection(
-         projectName,
-         connectionName,
-      );
-      return dbConnection;
+      // Prefer the in-memory API connection (which was materialized by the
+      // project on load and carries `attributes`). The DB row stores the
+      // raw config and FK columns, which aren't the ApiConnection shape.
+      const project = await this.projectStore.getProject(projectName, false);
+      return project.getApiConnection(connectionName);
    }
 
    public async listConnections(projectName: string): Promise<ApiConnection[]> {
@@ -231,22 +234,26 @@ export class ConnectionController {
          projectName,
          connectionName,
       );
-
       try {
+         const schema = await (
+            malloyConnection as Connection & {
+               fetchSelectSchema: (params: {
+                  connection: string;
+                  selectStr: string;
+               }) => Promise<unknown>;
+            }
+         ).fetchSelectSchema({
+            connection: connectionName,
+            selectStr: sqlStatement,
+         });
+
+         // BigQueryConnection returns `error.message` as a string on failure instead of throwing.
+         if (typeof schema === "string") {
+            throw new ConnectionError(schema);
+         }
+
          return {
-            source: JSON.stringify(
-               await (
-                  malloyConnection as Connection & {
-                     fetchSelectSchema: (params: {
-                        connection: string;
-                        selectStr: string;
-                     }) => Promise<unknown>;
-                  }
-               ).fetchSelectSchema({
-                  connection: connectionName,
-                  selectStr: sqlStatement,
-               }),
-            ),
+            source: JSON.stringify(schema),
          };
       } catch (error) {
          throw new ConnectionError((error as Error).message);

package/src/controller/manifest.controller.ts

@@ -0,0 +1,29 @@
+import { ManifestService } from "../service/manifest_service";
+import { ProjectStore } from "../service/project_store";
+import { resolveProjectId } from "../service/resolve_project";
+
+export class ManifestController {
+   constructor(
+      private projectStore: ProjectStore,
+      private manifestService: ManifestService,
+   ) {}
+
+   async getManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      // Verify the package exists so we return 404 instead of an empty manifest.
+      const project = await this.projectStore.getProject(projectName, false);
+      await project.getPackage(packageName, false);
+      return this.manifestService.getManifest(projectId, packageName);
+   }
+
+   async reloadManifest(projectName: string, packageName: string) {
+      const repository = this.projectStore.storageManager.getRepository();
+      const projectId = await resolveProjectId(repository, projectName);
+      return this.manifestService.reloadManifest(
+         projectId,
+         packageName,
+         projectName,
+      );
+   }
+}

package/src/controller/materialization.controller.ts

@@ -0,0 +1,125 @@
+import { BadRequestError } from "../errors";
+import { MaterializationService } from "../service/materialization_service";
+
+export class MaterializationController {
+   constructor(private materializationService: MaterializationService) {}
+
+   async createMaterialization(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateCreateBody(body);
+      return this.materializationService.createMaterialization(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateCreateBody(body: Record<string, unknown>): {
+      forceRefresh?: boolean;
+      autoLoadManifest?: boolean;
+   } {
+      const result: { forceRefresh?: boolean; autoLoadManifest?: boolean } = {};
+      if (body.forceRefresh !== undefined) {
+         if (typeof body.forceRefresh !== "boolean") {
+            throw new BadRequestError("forceRefresh must be a boolean");
+         }
+         result.forceRefresh = body.forceRefresh;
+      }
+      if (body.autoLoadManifest !== undefined) {
+         if (typeof body.autoLoadManifest !== "boolean") {
+            throw new BadRequestError("autoLoadManifest must be a boolean");
+         }
+         result.autoLoadManifest = body.autoLoadManifest;
+      }
+      return result;
+   }
+
+   async startMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.startMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async stopMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.stopMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async listMaterializations(
+      projectName: string,
+      packageName: string,
+      options?: { limit?: number; offset?: number },
+   ) {
+      return this.materializationService.listMaterializations(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   async getMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.getMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async deleteMaterialization(
+      projectName: string,
+      packageName: string,
+      materializationId: string,
+   ) {
+      return this.materializationService.deleteMaterialization(
+         projectName,
+         packageName,
+         materializationId,
+      );
+   }
+
+   async teardownPackage(
+      projectName: string,
+      packageName: string,
+      body: Record<string, unknown>,
+   ) {
+      const options = this.validateTeardownBody(body);
+      return this.materializationService.teardownPackage(
+         projectName,
+         packageName,
+         options,
+      );
+   }
+
+   private validateTeardownBody(body: Record<string, unknown>): {
+      dryRun?: boolean;
+   } {
+      const options: { dryRun?: boolean } = {};
+      if (body.dryRun !== undefined) {
+         if (typeof body.dryRun !== "boolean") {
+            throw new BadRequestError("dryRun must be a boolean");
+         }
+         options.dryRun = body.dryRun;
+      }
+      return options;
+   }
+}

package/src/controller/model.controller.ts

@@ -1,13 +1,12 @@
 import { components } from "../api";
 import { ModelNotFoundError } from "../errors";
 import { ProjectStore } from "../service/project_store";
+import type { FilterParams } from "../service/filter";
 
 type ApiNotebook = components["schemas"]["Notebook"];
 type ApiModel = components["schemas"]["Model"];
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
-export type ListModelsFilterEnum =
-   components["parameters"]["ListModelsFilterEnum"];
 export class ModelController {
    private projectStore: ProjectStore;
 
@@ -84,6 +83,8 @@ export class ModelController {
       packageName: string,
       notebookPath: string,
       cellIndex: number,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
    ): Promise<{
       type: "code" | "markdown";
       text: string;
@@ -101,6 +102,6 @@ export class ModelController {
          throw new ModelNotFoundError(`${notebookPath} is a model`);
       }
 
-      return model.executeNotebookCell(cellIndex);
+      return model.executeNotebookCell(cellIndex, filterParams, bypassFilters);
    }
 }

package/src/controller/package.controller.ts

@@ -2,15 +2,19 @@ import * as path from "path";
 import { components } from "../api";
 import { PUBLISHER_DATA_DIR } from "../constants";
 import { BadRequestError, FrozenConfigError } from "../errors";
+import { logger } from "../logger";
+import { ManifestService } from "../service/manifest_service";
 import { ProjectStore } from "../service/project_store";
 
 type ApiPackage = components["schemas"]["Package"];
 
 export class PackageController {
    private projectStore: ProjectStore;
+   private manifestService: ManifestService;
 
-   constructor(projectStore: ProjectStore) {
+   constructor(projectStore: ProjectStore, manifestService: ManifestService) {
       this.projectStore = projectStore;
+      this.manifestService = manifestService;
    }
 
    public async listPackages(projectName: string): Promise<ApiPackage[]> {
@@ -32,7 +36,11 @@ export class PackageController {
       return _package.getPackageMetadata();
    }
 
-   async addPackage(projectName: string, body: ApiPackage) {
+   async addPackage(
+      projectName: string,
+      body: ApiPackage,
+      options?: { autoLoadManifest?: boolean },
+   ) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();
       }
@@ -46,9 +54,52 @@ export class PackageController {
       const result = await project.addPackage(body.name);
       await this.projectStore.addPackageToDatabase(projectName, body.name);
 
+      if (options?.autoLoadManifest === true) {
+         await this.tryLoadExistingManifest(projectName, body.name);
+      }
+
       return result;
    }
 
+   /**
+    * If there are already manifest entries for this package (e.g. from a
+    * previous materialization run), reload all models with the manifest so
+    * persist references resolve to the materialized tables immediately.
+    */
+   private async tryLoadExistingManifest(
+      projectName: string,
+      packageName: string,
+   ): Promise<void> {
+      try {
+         const repository = this.projectStore.storageManager.getRepository();
+         const dbProject = await repository.getProjectByName(projectName);
+         if (!dbProject) return;
+
+         const manifest = await this.manifestService.getManifest(
+            dbProject.id,
+            packageName,
+         );
+         if (Object.keys(manifest.entries).length === 0) return;
+
+         await this.manifestService.reloadManifest(
+            dbProject.id,
+            packageName,
+            projectName,
+         );
+         logger.info("Auto-loaded existing manifest for added package", {
+            projectName,
+            packageName,
+            entryCount: Object.keys(manifest.entries).length,
+         });
+      } catch (error) {
+         logger.warn("Failed to auto-load manifest for package", {
+            projectName,
+            packageName,
+            error,
+         });
+      }
+   }
+
    public async deletePackage(projectName: string, packageName: string) {
       if (this.projectStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();

package/src/controller/query.controller.ts

@@ -3,6 +3,7 @@ import { components } from "../api";
 import { API_PREFIX } from "../constants";
 import { ModelNotFoundError } from "../errors";
 import { ProjectStore } from "../service/project_store";
+import type { FilterParams } from "../service/filter";
 
 type ApiQuery = components["schemas"]["QueryResult"];
 
@@ -29,6 +30,8 @@ export class QueryController {
       queryName: string,
       query: string,
       compactJson: boolean = false,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
    ): Promise<ApiQuery> {
       const project = await this.projectStore.getProject(projectName, false);
       const p = await project.getPackage(packageName, false);
@@ -41,6 +44,8 @@ export class QueryController {
             sourceName,
             queryName,
             query,
+            filterParams,
+            bypassFilters,
          );
          const renderLogs = validateRenderTags(result);
          return {

package/src/errors.ts

@@ -20,6 +20,12 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
       return httpError(502, error.message);
+   } else if (error instanceof MaterializationNotFoundError) {
+      return httpError(404, error.message);
+   } else if (error instanceof MaterializationConflictError) {
+      return httpError(409, error.message);
+   } else if (error instanceof InvalidStateTransitionError) {
+      return httpError(409, error.message);
    } else {
       return httpError(500, error.message);
    }
@@ -90,3 +96,21 @@ export class FrozenConfigError extends Error {
       super(message);
    }
 }
+
+export class MaterializationNotFoundError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class MaterializationConflictError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
+export class InvalidStateTransitionError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}

package/src/mcp/prompts/handlers.ts

@@ -1,4 +1,4 @@
-import * as Handlebars from "handlebars";
+import Handlebars from "handlebars";
 import { z } from "zod";
 import {
    PromptMessageSchema,

package/src/mcp/resources/model_resource.ts

@@ -82,15 +82,18 @@ export function registerModelResource(
                   const compiledModelDefinition: components["schemas"]["CompiledModel"] =
                      await modelInstance.getModel();
 
-                  // Return the definition, potentially adding identifiers back if needed
-                  // (Check if compiledModelDefinition already includes them)
-                  return {
-                     ...compiledModelDefinition,
-                     // Ensure these are present if not already in compiledModelDefinition
-                     // path: modelPath,
-                     // packageName: packageName,
-                     // projectName: projectName
-                  };
+                  // Strip implicit filters from agent-facing responses
+                  if (compiledModelDefinition.sources) {
+                     for (const source of compiledModelDefinition.sources) {
+                        if (source.filters) {
+                           source.filters = source.filters.filter(
+                              (f) => !f.implicit,
+                           );
+                        }
+                     }
+                  }
+
+                  return compiledModelDefinition;
                } catch (error) {
                   let errorDetails;
                   // Provide specific context for error messages

package/src/mcp/resources/source_resource.ts

@@ -2,8 +2,8 @@ import {
    McpServer,
    ResourceTemplate,
 } from "@modelcontextprotocol/sdk/server/mcp.js";
-import { URL } from "url"; // Import URL
-import type { components } from "../../api"; // Need this for Source definition type
+import { URL } from "url";
+import type { components } from "../../api";
 import { ModelCompilationError } from "../../errors";
 import { logger } from "../../logger";
 import { ProjectStore } from "../../service/project_store";
@@ -89,22 +89,23 @@ export function registerSourceResource(
                   if (!sources) {
                      throw new Error("Could not retrieve sources from model.");
                   }
-                  // Add type annotation for 's'
                   const source = sources.find(
-                     // @ts-expect-error TODO: Fix missing Source type in API
                      (s: components["schemas"]["Source"]) =>
                         s.name === sourceName,
                   );
 
                   if (!source) {
-                     // Specific "Source not found" error
                      const errorDetails = getNotFoundError(
                         `Source '${sourceName}' in model '${modelPath}' package '${packageName}' project '${projectName}'`,
                      );
                      throw new McpGetResourceError(errorDetails);
                   }
 
-                  // Return the source definition
+                  // Strip implicit filters from agent-facing responses
+                  if (source.filters) {
+                     source.filters = source.filters.filter((f) => !f.implicit);
+                  }
+
                   return source;
                } catch (error) {
                   // Catch errors from getModelForQuery or finding the source

package/src/mcp/resources/view_resource.ts

@@ -83,7 +83,6 @@ export function registerViewResource(
                      throw new Error("Could not retrieve sources from model.");
                   }
                   const source = sources.find(
-                     // @ts-expect-error TODO: Fix missing Source type in API
                      (s: components["schemas"]["Source"]) =>
                         s.name === sourceName,
                   );

package/src/mcp/tools/execute_query_tool.ts

@@ -24,6 +24,12 @@ const executeQueryShape = {
    query: z.string().optional().describe("Ad-hoc Malloy query code"),
    sourceName: z.string().optional().describe("Source name for a view"),
    queryName: z.string().optional().describe("Named query or view"),
+   filterParams: z
+      .record(z.union([z.string(), z.array(z.string())]))
+      .optional()
+      .describe(
+         "Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations.",
+      ),
 };
 
 // Type inference is handled automatically by the MCP server based on the executeQueryShape
@@ -49,6 +55,7 @@ export function registerExecuteQueryTool(
             query,
             sourceName,
             queryName,
+            filterParams,
          } = params;
 
          logger.info("[MCP Tool executeQuery] Received params:", { params });
@@ -120,6 +127,7 @@ export function registerExecuteQueryTool(
                   undefined,
                   undefined,
                   query,
+                  filterParams,
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"
@@ -165,6 +173,7 @@ export function registerExecuteQueryTool(
                   sourceName,
                   queryName,
                   undefined,
+                  filterParams,
                );
                const { validateRenderTags } = await import(
                   "@malloydata/render-validator"