@malloy-publisher/server 0.0.181 → 0.0.183-dev

package/src/service/model.ts

@@ -22,7 +22,7 @@ import {
    MalloySQLParser,
    MalloySQLStatementType,
 } from "@malloydata/malloy-sql";
-import malloyPackage from "@malloydata/malloy/package.json";
+import { createRequire } from "module";
 import { DataStyles } from "@malloydata/render";
 import { metrics } from "@opentelemetry/api";
 import * as fs from "fs/promises";
@@ -41,13 +41,22 @@ import {
    ModelNotFoundError,
 } from "../errors";
 import { logger } from "../logger";
+import { BuildManifest } from "../storage/DatabaseInterface";
 import { URL_READER } from "../utils";
+import {
+   buildFilterClause,
+   FilterValidationError,
+   injectFilterRefinement,
+   parseFilters,
+   type FilterDefinition,
+   type FilterParams,
+} from "./filter";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
-// @ts-expect-error TODO: Fix missing Source type in API
 type ApiSource = components["schemas"]["Source"];
+type ApiFilter = components["schemas"]["Filter"];
 type ApiView = components["schemas"]["View"];
 type ApiQuery = components["schemas"]["Query"];
 export type ApiConnection = components["schemas"]["Connection"];
@@ -56,7 +65,11 @@ export type PostgresConnection = components["schemas"]["PostgresConnection"];
 export type BigqueryConnection = components["schemas"]["BigqueryConnection"];
 export type TrinoConnection = components["schemas"]["TrinoConnection"];
 
-const MALLOY_VERSION = malloyPackage.version;
+const MALLOY_VERSION = (
+   createRequire(import.meta.url)("@malloydata/malloy/package.json") as {
+      version: string;
+   }
+).version;
 
 export type ModelType = "model" | "notebook";
 
@@ -64,6 +77,8 @@ interface RunnableNotebookCell {
    type: "code" | "markdown";
    text: string;
    runnable?: QueryMaterializer;
+   /** Retained so we can rebuild the query with filter refinements at execution time. */
+   modelMaterializer?: ModelMaterializer;
    newSources?: Malloy.SourceInfo[];
    queryInfo?: Malloy.QueryInfo;
 }
@@ -81,6 +96,8 @@ export class Model {
    private sourceInfos: Malloy.SourceInfo[] | undefined;
    private runnableNotebookCells: RunnableNotebookCell[] | undefined;
    private compilationError: MalloyError | Error | undefined;
+   /** Parsed #(filter) definitions keyed by source name. */
+   private filterMap: Map<string, FilterDefinition[]>;
    private meter = metrics.getMeter("publisher");
    private queryExecutionHistogram = this.meter.createHistogram(
       "malloy_model_query_duration",
@@ -103,6 +120,7 @@ export class Model {
       sourceInfos: Malloy.SourceInfo[] | undefined,
       runnableNotebookCells: RunnableNotebookCell[] | undefined,
       compilationError: MalloyError | Error | undefined,
+      filterMap?: Map<string, FilterDefinition[]>,
    ) {
       this.packageName = packageName;
       this.modelPath = modelPath;
@@ -115,21 +133,47 @@ export class Model {
       this.sourceInfos = sourceInfos;
       this.runnableNotebookCells = runnableNotebookCells;
       this.compilationError = compilationError;
+      this.filterMap = filterMap ?? new Map();
       this.modelInfo = this.modelDef
          ? modelDefToModelInfo(this.modelDef)
          : undefined;
    }
 
+   /**
+    * Get the parsed filter definitions for a given source name.
+    * Returns an empty array if no filters are declared.
+    */
+   public getFilters(sourceName: string): FilterDefinition[] {
+      return this.filterMap.get(sourceName) ?? [];
+   }
+
+   /**
+    * Best-effort extraction of a source name from an ad-hoc Malloy query string.
+    * Matches patterns like `run: source_name -> ...` or `source_name -> ...`.
+    */
+   private extractSourceName(query?: string): string | undefined {
+      if (!query) return undefined;
+      const runMatch = query.match(/run\s*:\s*(\w+)\s*->/);
+      const arrowMatch = query.match(/^\s*(\w+)\s*->/m);
+      return runMatch?.[1] ?? arrowMatch?.[1];
+   }
+
    public static async create(
       packageName: string,
       packagePath: string,
       modelPath: string,
       connections: Map<string, Connection>,
+      options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<Model> {
       // getModelRuntime might throw a ModelNotFoundError. It's the callers responsibility
       // to pass a valid model path or handle the error.
       const { runtime, modelURL, importBaseURL, dataStyles, modelType } =
-         await Model.getModelRuntime(packagePath, modelPath, connections);
+         await Model.getModelRuntime(
+            packagePath,
+            modelPath,
+            connections,
+            options,
+         );
 
       try {
          const { modelMaterializer, runnableNotebookCells } =
@@ -143,10 +187,13 @@ export class Model {
          let modelDef = undefined;
          let sources = undefined;
          let queries = undefined;
+         let filterMap: Map<string, FilterDefinition[]> | undefined;
          const sourceInfos: Malloy.SourceInfo[] = [];
          if (modelMaterializer) {
             modelDef = (await modelMaterializer.getModel())._modelDef;
-            sources = Model.getSources(modelPath, modelDef);
+            const sourceResult = Model.getSources(modelPath, modelDef);
+            sources = sourceResult.sources;
+            filterMap = sourceResult.filterMap;
             queries = Model.getQueries(modelPath, modelDef);
 
             // Collect sourceInfos from imported models first
@@ -207,6 +254,7 @@ export class Model {
             sourceInfos.length > 0 ? sourceInfos : undefined,
             runnableNotebookCells,
             undefined,
+            filterMap,
          );
       } catch (error) {
          let computedError = error;
@@ -254,7 +302,7 @@ export class Model {
    }
 
    public getQueries(): ApiQuery[] | undefined {
-      return this.sources;
+      return this.queries;
    }
 
    public async getModel(): Promise<ApiCompiledModel> {
@@ -292,6 +340,8 @@ export class Model {
       sourceName?: string,
       queryName?: string,
       query?: string,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
@@ -318,12 +368,11 @@ export class Model {
 
       // Wrap loadQuery calls in try-catch to handle query parsing errors
       try {
+         let queryString: string;
          if (!sourceName && !queryName && query) {
-            runnable = this.modelMaterializer.loadQuery("\n" + query);
+            queryString = "\n" + query;
          } else if (queryName && !query) {
-            runnable = this.modelMaterializer.loadQuery(
-               `\nrun: ${sourceName ? sourceName + "->" : ""}${queryName}`,
-            );
+            queryString = `\nrun: ${sourceName ? sourceName + "->" : ""}${queryName}`;
          } else {
             const endTime = performance.now();
             const executionTime = endTime - startTime;
@@ -338,11 +387,35 @@ export class Model {
                "Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.",
             );
          }
+
+         // Inject source filter predicates unless bypassed
+         if (!bypassFilters) {
+            const effectiveSource = sourceName ?? this.extractSourceName(query);
+            if (effectiveSource) {
+               const filters = this.getFilters(effectiveSource);
+               if (filters.length > 0) {
+                  const filterClause = buildFilterClause(
+                     filters,
+                     filterParams ?? {},
+                  );
+                  queryString = injectFilterRefinement(
+                     queryString,
+                     filterClause,
+                  );
+               }
+            }
+         }
+
+         runnable = this.modelMaterializer.loadQuery(queryString);
       } catch (error) {
          // Re-throw BadRequestError as-is
          if (error instanceof BadRequestError) {
             throw error;
          }
+         // Source filter validation errors are client errors (400)
+         if (error instanceof FilterValidationError) {
+            throw new BadRequestError(error.message);
+         }
          // Re-throw MalloyError as-is (maps to 400)
          if (error instanceof MalloyError) {
             throw error;
@@ -491,7 +564,11 @@ export class Model {
       } as ApiRawNotebook;
    }
 
-   public async executeNotebookCell(cellIndex: number): Promise<{
+   public async executeNotebookCell(
+      cellIndex: number,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
+   ): Promise<{
       type: "code" | "markdown";
       text: string;
       queryName?: string;
@@ -527,18 +604,44 @@ export class Model {
 
       if (cell.runnable) {
          try {
+            let runnableToExecute = cell.runnable;
+
+            // If filters need to be applied, rebuild the query with a refinement
+            if (!bypassFilters && cell.modelMaterializer) {
+               const effectiveSource = this.extractSourceName(cell.text);
+               if (effectiveSource) {
+                  const filters = this.getFilters(effectiveSource);
+                  if (filters.length > 0) {
+                     const filterClause = buildFilterClause(
+                        filters,
+                        filterParams ?? {},
+                     );
+                     if (filterClause) {
+                        const refinedQuery = injectFilterRefinement(
+                           cell.text,
+                           filterClause,
+                        );
+                        runnableToExecute =
+                           cell.modelMaterializer.loadQuery(refinedQuery);
+                     }
+                  }
+               }
+            }
+
             const rowLimit =
-               (await cell.runnable.getPreparedResult()).resultExplore.limit ||
-               ROW_LIMIT;
-            const result = await cell.runnable.run({ rowLimit });
-            const query = (await cell.runnable.getPreparedQuery())._query;
+               (await runnableToExecute.getPreparedResult()).resultExplore
+                  .limit || ROW_LIMIT;
+            const result = await runnableToExecute.run({ rowLimit });
+            const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;
             queryResult =
                result?._queryResult &&
                this.modelInfo &&
                JSON.stringify(API.util.wrapResult(result));
          } catch (error) {
-            // Re-throw execution errors so the client knows about them
+            if (error instanceof FilterValidationError) {
+               throw new BadRequestError(error.message);
+            }
             if (error instanceof MalloyError) {
                throw error;
             }
@@ -570,6 +673,7 @@ export class Model {
       packagePath: string,
       modelPath: string,
       connections: Map<string, Connection>,
+      options?: { buildManifest?: BuildManifest["entries"] },
    ): Promise<{
       runtime: Runtime;
       modelURL: URL;
@@ -609,10 +713,23 @@ export class Model {
          `SET FILE_SEARCH_PATH='${workingDirectory}';`,
       );
 
-      const runtime = new Runtime({
+      const runtimeOptions: {
+         urlReader: typeof urlReader;
+         connections: FixedConnectionMap;
+         buildManifest?: BuildManifest;
+      } = {
          urlReader,
          connections: new FixedConnectionMap(connections, "duckdb"),
-      });
+      };
+
+      if (options?.buildManifest) {
+         runtimeOptions.buildManifest = {
+            entries: options.buildManifest,
+            strict: false,
+         };
+      }
+
+      const runtime = new Runtime(runtimeOptions);
       const dataStyles = urlReader.getHackyAccumulatedDataStyles();
       return { runtime, modelURL, importBaseURL, dataStyles, modelType };
    }
@@ -644,38 +761,98 @@ export class Model {
    private static getSources(
       modelPath: string,
       modelDef: ModelDef,
-   ): ApiSource[] {
-      return Object.values(modelDef.contents)
+   ): {
+      sources: ApiSource[];
+      filterMap: Map<string, FilterDefinition[]>;
+   } {
+      const filterMap = new Map<string, FilterDefinition[]>();
+
+      const sources = Object.values(modelDef.contents)
          .filter((obj) => isSourceDef(obj))
-         .map(
-            (sourceObj) =>
-               ({
-                  name: sourceObj.as || sourceObj.name,
-                  annotations: (sourceObj as StructDef).annotation?.blockNotes
-                     ?.filter((note) => note.at.url.includes(modelPath))
-                     .map((note) => note.text),
-                  views: (sourceObj as StructDef).fields
-                     .filter((turtleObj) => turtleObj.type === "turtle")
-                     .filter((turtleObj) =>
-                        // TODO(kjnesbit): Fix non-reduce views. Filter out
-                        // non-reduce views, i.e., indexes. Need to discuss with Will.
-                        (turtleObj as TurtleDef).pipeline
-                           .map((stage) => stage.type)
-                           .every((type) => type == "reduce"),
-                     )
-                     .map(
-                        (turtleObj) =>
-                           ({
-                              name: turtleObj.as || turtleObj.name,
-                              annotations: turtleObj?.annotation?.blockNotes
-                                 ?.filter((note) =>
-                                    note.at.url.includes(modelPath),
-                                 )
-                                 .map((note) => note.text),
-                           }) as ApiView,
-                     ),
-               }) as ApiSource,
-         );
+         .map((sourceObj) => {
+            const sourceName = sourceObj.as || sourceObj.name;
+            const annotations = (sourceObj as StructDef).annotation?.blockNotes
+               ?.filter((note) => note.at.url.includes(modelPath))
+               .map((note) => note.text);
+
+            // Parse #(filter) from ALL annotations, traversing the inherits
+            // chain so that filters on a base source (e.g. `recalls`) are
+            // picked up by an extending source (`manufacturer_recalls is
+            // recalls extend {}`).  The Malloy compiler stores the base
+            // source's annotations in `annotation.inherits`.
+            //
+            // The chain goes child → parent, so we collect child-first.
+            // parseFilters uses "last wins" dedup, so we reverse to put
+            // parent annotations first and child annotations last (winning).
+            const collectedAnnotations: string[][] = [];
+            let curAnnotation: Annotation | undefined = (sourceObj as StructDef)
+               .annotation;
+            while (curAnnotation) {
+               if (curAnnotation.blockNotes) {
+                  collectedAnnotations.push(
+                     curAnnotation.blockNotes.map((note) => note.text),
+                  );
+               }
+               curAnnotation = curAnnotation.inherits;
+            }
+            const allAnnotations = collectedAnnotations.reverse().flat();
+            let filters: ApiFilter[] | undefined;
+            if (allAnnotations.length > 0) {
+               try {
+                  const parsed = parseFilters(allAnnotations);
+                  if (parsed.length > 0) {
+                     filterMap.set(sourceName, parsed);
+                     const structFields = (sourceObj as StructDef).fields;
+                     filters = parsed.map((f) => {
+                        const field = structFields.find(
+                           (fd) => (fd.as || fd.name) === f.dimension,
+                        );
+                        return {
+                           name: f.name,
+                           dimension: f.dimension,
+                           type: f.type,
+                           implicit: f.implicit,
+                           required: f.required,
+                           dimensionType: field?.type as string | undefined,
+                        };
+                     });
+                  }
+               } catch (err) {
+                  logger.warn(
+                     `Failed to parse filter annotations on source "${sourceName}"`,
+                     { error: err },
+                  );
+               }
+            }
+
+            const views = (sourceObj as StructDef).fields
+               .filter((turtleObj) => turtleObj.type === "turtle")
+               .filter((turtleObj) =>
+                  // TODO(kjnesbit): Fix non-reduce views. Filter out
+                  // non-reduce views, i.e., indexes. Need to discuss with Will.
+                  (turtleObj as TurtleDef).pipeline
+                     .map((stage) => stage.type)
+                     .every((type) => type == "reduce"),
+               )
+               .map(
+                  (turtleObj) =>
+                     ({
+                        name: turtleObj.as || turtleObj.name,
+                        annotations: turtleObj?.annotation?.blockNotes
+                           ?.filter((note) => note.at.url.includes(modelPath))
+                           .map((note) => note.text),
+                     }) as ApiView,
+               );
+
+            return {
+               name: sourceName,
+               annotations,
+               views,
+               filters,
+            } as ApiSource;
+         });
+
+      return { sources, filterMap };
    }
 
    static async getModelMaterializer(
@@ -857,6 +1034,7 @@ export class Model {
                      type: "code",
                      text: stmt.text,
                      runnable: runnable,
+                     modelMaterializer: localMM,
                      newSources,
                      queryInfo,
                   } as RunnableNotebookCell;

package/src/service/package.ts

@@ -19,6 +19,7 @@ import {
 } from "../constants";
 import { PackageNotFoundError } from "../errors";
 import { formatDuration, logger } from "../logger";
+import { BuildManifest } from "../storage/DatabaseInterface";
 import { Model } from "./model";
 
 type ApiDatabase = components["schemas"]["Database"];
@@ -189,6 +190,10 @@ export class Package {
       return this.packageName;
    }
 
+   public getPackagePath(): string {
+      return this.packagePath;
+   }
+
    public getPackageMetadata(): ApiPackage {
       return this.packageMetadata;
    }
@@ -201,6 +206,51 @@ export class Package {
       return this.models.get(modelPath);
    }
 
+   public getModelPaths(): string[] {
+      return Array.from(this.models.keys());
+   }
+
+   /**
+    * Recompile every model in the package with the given build manifest
+    * so queries resolve persist references to materialized tables.
+    *
+    * Builds a fresh map off to the side and swaps it in at the end. If any
+    * recompile fails the whole call rejects before the swap and the live
+    * `this.models` reference remains untouched — no half-loaded state is
+    * ever observable to concurrent readers.
+    */
+   public async reloadAllModels(
+      buildManifest: BuildManifest["entries"],
+   ): Promise<void> {
+      const modelPaths = Array.from(this.models.keys());
+      logger.info("Reloading all models with build manifest", {
+         packageName: this.packageName,
+         modelCount: modelPaths.length,
+         manifestEntryCount: Object.keys(buildManifest).length,
+      });
+
+      const reloaded = await Promise.all(
+         modelPaths.map((modelPath) =>
+            Model.create(
+               this.packageName,
+               this.packagePath,
+               modelPath,
+               this.connections,
+               { buildManifest },
+            ),
+         ),
+      );
+      const nextModels = new Map<string, Model>();
+      for (const model of reloaded) {
+         nextModels.set(model.getPath(), model);
+      }
+      this.models = nextModels;
+   }
+
+   public getConnections(): Map<string, Connection> {
+      return this.connections;
+   }
+
    public getMalloyConnection(connectionName: string): Connection {
       const connection = this.connections.get(connectionName);
       if (!connection) {

package/src/service/project_store.ts

@@ -320,6 +320,7 @@ export class ProjectStore {
       };
       const existingProject = await repository.getProjectByName(projectName);
 
+      let dbProject: { id: string; name: string };
       if (existingProject) {
          const updateData = {
             description: projectDescription,
@@ -327,10 +328,27 @@ export class ProjectStore {
          };
 
          await repository.updateProject(existingProject.id, updateData);
-         return { id: existingProject.id, name: projectName };
+         dbProject = { id: existingProject.id, name: projectName };
       } else {
-         return await repository.createProject(projectData);
+         dbProject = await repository.createProject(projectData);
       }
+
+      // Initialize DuckLake manifest storage if configured on the project.
+      const materializationStorage = project.metadata
+         ?.materializationStorage as
+         | { catalogUrl?: string; dataPath?: string }
+         | undefined;
+      if (
+         materializationStorage?.catalogUrl &&
+         materializationStorage?.dataPath
+      ) {
+         await this.storageManager.initializeDuckLakeForProject(dbProject.id, {
+            catalogUrl: materializationStorage.catalogUrl,
+            dataPath: materializationStorage.dataPath,
+         });
+      }
+
+      return dbProject;
    }
 
    private async addPackages(
@@ -917,6 +935,7 @@ export class ProjectStore {
    private isLocalPath(location: string) {
       return (
          location.startsWith("./") ||
+         location.startsWith("../") ||
          location.startsWith("~/") ||
          location.startsWith("/") ||
          path.isAbsolute(location)

package/src/service/quoting.ts

@@ -0,0 +1,41 @@
+/**
+ * Minimal identifier-quoting surface. Every `Dialect` in `@malloydata/malloy`
+ * implements this; we accept the duck type so tests can inject a fake without
+ * instantiating a full dialect.
+ */
+export interface Quoter {
+   quoteTablePath(seg: string): string;
+}
+
+/**
+ * Quote a potentially schema-qualified table path (e.g. "schema.table")
+ * by quoting each segment individually with the dialect's quoteTablePath.
+ */
+export function quoteTablePath(path: string, dialect: Quoter): string {
+   return path
+      .split(".")
+      .map((seg) => dialect.quoteTablePath(seg))
+      .join(".");
+}
+
+/**
+ * Split a possibly schema-qualified table name into its schema prefix
+ * (including the trailing dot) and the bare table name.
+ *
+ * Examples:
+ *   "my_schema.my_table" -> { schemaPrefix: "my_schema.", bareName: "my_table" }
+ *   "my_table"           -> { schemaPrefix: "", bareName: "my_table" }
+ */
+export function splitTablePath(tableName: string): {
+   schemaPrefix: string;
+   bareName: string;
+} {
+   const lastDot = tableName.lastIndexOf(".");
+   if (lastDot >= 0) {
+      return {
+         schemaPrefix: tableName.substring(0, lastDot + 1),
+         bareName: tableName.substring(lastDot + 1),
+      };
+   }
+   return { schemaPrefix: "", bareName: tableName };
+}

package/src/service/resolve_project.ts

@@ -0,0 +1,13 @@
+import { ProjectNotFoundError } from "../errors";
+import { ResourceRepository } from "../storage/DatabaseInterface";
+
+export async function resolveProjectId(
+   repository: ResourceRepository,
+   projectName: string,
+): Promise<string> {
+   const dbProject = await repository.getProjectByName(projectName);
+   if (!dbProject) {
+      throw new ProjectNotFoundError(`Project '${projectName}' not found`);
+   }
+   return dbProject.id;
+}