@malloy-publisher/server 0.0.180 → 0.0.182

package/src/service/model.ts

@@ -42,12 +42,20 @@ import {
 } from "../errors";
 import { logger } from "../logger";
 import { URL_READER } from "../utils";
+import {
+   buildFilterClause,
+   injectFilterRefinement,
+   parseFilters,
+   FilterValidationError,
+   type FilterDefinition,
+   type FilterParams,
+} from "./filter";
 
 type ApiCompiledModel = components["schemas"]["CompiledModel"];
 type ApiNotebookCell = components["schemas"]["NotebookCell"];
 type ApiRawNotebook = components["schemas"]["RawNotebook"];
-// @ts-expect-error TODO: Fix missing Source type in API
 type ApiSource = components["schemas"]["Source"];
+type ApiFilter = components["schemas"]["Filter"];
 type ApiView = components["schemas"]["View"];
 type ApiQuery = components["schemas"]["Query"];
 export type ApiConnection = components["schemas"]["Connection"];
@@ -64,6 +72,8 @@ interface RunnableNotebookCell {
    type: "code" | "markdown";
    text: string;
    runnable?: QueryMaterializer;
+   /** Retained so we can rebuild the query with filter refinements at execution time. */
+   modelMaterializer?: ModelMaterializer;
    newSources?: Malloy.SourceInfo[];
    queryInfo?: Malloy.QueryInfo;
 }
@@ -81,6 +91,8 @@ export class Model {
    private sourceInfos: Malloy.SourceInfo[] | undefined;
    private runnableNotebookCells: RunnableNotebookCell[] | undefined;
    private compilationError: MalloyError | Error | undefined;
+   /** Parsed #(filter) definitions keyed by source name. */
+   private filterMap: Map<string, FilterDefinition[]>;
    private meter = metrics.getMeter("publisher");
    private queryExecutionHistogram = this.meter.createHistogram(
       "malloy_model_query_duration",
@@ -103,6 +115,7 @@ export class Model {
       sourceInfos: Malloy.SourceInfo[] | undefined,
       runnableNotebookCells: RunnableNotebookCell[] | undefined,
       compilationError: MalloyError | Error | undefined,
+      filterMap?: Map<string, FilterDefinition[]>,
    ) {
       this.packageName = packageName;
       this.modelPath = modelPath;
@@ -115,11 +128,31 @@ export class Model {
       this.sourceInfos = sourceInfos;
       this.runnableNotebookCells = runnableNotebookCells;
       this.compilationError = compilationError;
+      this.filterMap = filterMap ?? new Map();
       this.modelInfo = this.modelDef
          ? modelDefToModelInfo(this.modelDef)
          : undefined;
    }
 
+   /**
+    * Get the parsed filter definitions for a given source name.
+    * Returns an empty array if no filters are declared.
+    */
+   public getFilters(sourceName: string): FilterDefinition[] {
+      return this.filterMap.get(sourceName) ?? [];
+   }
+
+   /**
+    * Best-effort extraction of a source name from an ad-hoc Malloy query string.
+    * Matches patterns like `run: source_name -> ...` or `source_name -> ...`.
+    */
+   private extractSourceName(query?: string): string | undefined {
+      if (!query) return undefined;
+      const runMatch = query.match(/run\s*:\s*(\w+)\s*->/);
+      const arrowMatch = query.match(/^\s*(\w+)\s*->/m);
+      return runMatch?.[1] ?? arrowMatch?.[1];
+   }
+
    public static async create(
       packageName: string,
       packagePath: string,
@@ -143,10 +176,13 @@ export class Model {
          let modelDef = undefined;
          let sources = undefined;
          let queries = undefined;
+         let filterMap: Map<string, FilterDefinition[]> | undefined;
          const sourceInfos: Malloy.SourceInfo[] = [];
          if (modelMaterializer) {
             modelDef = (await modelMaterializer.getModel())._modelDef;
-            sources = Model.getSources(modelPath, modelDef);
+            const sourceResult = Model.getSources(modelPath, modelDef);
+            sources = sourceResult.sources;
+            filterMap = sourceResult.filterMap;
             queries = Model.getQueries(modelPath, modelDef);
 
             // Collect sourceInfos from imported models first
@@ -207,6 +243,7 @@ export class Model {
             sourceInfos.length > 0 ? sourceInfos : undefined,
             runnableNotebookCells,
             undefined,
+            filterMap,
          );
       } catch (error) {
          let computedError = error;
@@ -292,6 +329,8 @@ export class Model {
       sourceName?: string,
       queryName?: string,
       query?: string,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
    ): Promise<{
       result: Malloy.Result;
       compactResult: QueryData;
@@ -318,12 +357,11 @@ export class Model {
 
       // Wrap loadQuery calls in try-catch to handle query parsing errors
       try {
+         let queryString: string;
          if (!sourceName && !queryName && query) {
-            runnable = this.modelMaterializer.loadQuery("\n" + query);
+            queryString = "\n" + query;
          } else if (queryName && !query) {
-            runnable = this.modelMaterializer.loadQuery(
-               `\nrun: ${sourceName ? sourceName + "->" : ""}${queryName}`,
-            );
+            queryString = `\nrun: ${sourceName ? sourceName + "->" : ""}${queryName}`;
          } else {
             const endTime = performance.now();
             const executionTime = endTime - startTime;
@@ -338,11 +376,35 @@ export class Model {
                "Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.",
             );
          }
+
+         // Inject source filter predicates unless bypassed
+         if (!bypassFilters) {
+            const effectiveSource = sourceName ?? this.extractSourceName(query);
+            if (effectiveSource) {
+               const filters = this.getFilters(effectiveSource);
+               if (filters.length > 0) {
+                  const filterClause = buildFilterClause(
+                     filters,
+                     filterParams ?? {},
+                  );
+                  queryString = injectFilterRefinement(
+                     queryString,
+                     filterClause,
+                  );
+               }
+            }
+         }
+
+         runnable = this.modelMaterializer.loadQuery(queryString);
       } catch (error) {
          // Re-throw BadRequestError as-is
          if (error instanceof BadRequestError) {
             throw error;
          }
+         // Source filter validation errors are client errors (400)
+         if (error instanceof FilterValidationError) {
+            throw new BadRequestError(error.message);
+         }
          // Re-throw MalloyError as-is (maps to 400)
          if (error instanceof MalloyError) {
             throw error;
@@ -491,7 +553,11 @@ export class Model {
       } as ApiRawNotebook;
    }
 
-   public async executeNotebookCell(cellIndex: number): Promise<{
+   public async executeNotebookCell(
+      cellIndex: number,
+      filterParams?: FilterParams,
+      bypassFilters?: boolean,
+   ): Promise<{
       type: "code" | "markdown";
       text: string;
       queryName?: string;
@@ -527,18 +593,44 @@ export class Model {
 
       if (cell.runnable) {
          try {
+            let runnableToExecute = cell.runnable;
+
+            // If filters need to be applied, rebuild the query with a refinement
+            if (!bypassFilters && cell.modelMaterializer) {
+               const effectiveSource = this.extractSourceName(cell.text);
+               if (effectiveSource) {
+                  const filters = this.getFilters(effectiveSource);
+                  if (filters.length > 0) {
+                     const filterClause = buildFilterClause(
+                        filters,
+                        filterParams ?? {},
+                     );
+                     if (filterClause) {
+                        const refinedQuery = injectFilterRefinement(
+                           cell.text,
+                           filterClause,
+                        );
+                        runnableToExecute =
+                           cell.modelMaterializer.loadQuery(refinedQuery);
+                     }
+                  }
+               }
+            }
+
             const rowLimit =
-               (await cell.runnable.getPreparedResult()).resultExplore.limit ||
-               ROW_LIMIT;
-            const result = await cell.runnable.run({ rowLimit });
-            const query = (await cell.runnable.getPreparedQuery())._query;
+               (await runnableToExecute.getPreparedResult()).resultExplore
+                  .limit || ROW_LIMIT;
+            const result = await runnableToExecute.run({ rowLimit });
+            const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;
             queryResult =
                result?._queryResult &&
                this.modelInfo &&
                JSON.stringify(API.util.wrapResult(result));
          } catch (error) {
-            // Re-throw execution errors so the client knows about them
+            if (error instanceof FilterValidationError) {
+               throw new BadRequestError(error.message);
+            }
             if (error instanceof MalloyError) {
                throw error;
             }
@@ -644,38 +736,82 @@ export class Model {
    private static getSources(
       modelPath: string,
       modelDef: ModelDef,
-   ): ApiSource[] {
-      return Object.values(modelDef.contents)
+   ): {
+      sources: ApiSource[];
+      filterMap: Map<string, FilterDefinition[]>;
+   } {
+      const filterMap = new Map<string, FilterDefinition[]>();
+
+      const sources = Object.values(modelDef.contents)
          .filter((obj) => isSourceDef(obj))
-         .map(
-            (sourceObj) =>
-               ({
-                  name: sourceObj.as || sourceObj.name,
-                  annotations: (sourceObj as StructDef).annotation?.blockNotes
-                     ?.filter((note) => note.at.url.includes(modelPath))
-                     .map((note) => note.text),
-                  views: (sourceObj as StructDef).fields
-                     .filter((turtleObj) => turtleObj.type === "turtle")
-                     .filter((turtleObj) =>
-                        // TODO(kjnesbit): Fix non-reduce views. Filter out
-                        // non-reduce views, i.e., indexes. Need to discuss with Will.
-                        (turtleObj as TurtleDef).pipeline
-                           .map((stage) => stage.type)
-                           .every((type) => type == "reduce"),
-                     )
-                     .map(
-                        (turtleObj) =>
-                           ({
-                              name: turtleObj.as || turtleObj.name,
-                              annotations: turtleObj?.annotation?.blockNotes
-                                 ?.filter((note) =>
-                                    note.at.url.includes(modelPath),
-                                 )
-                                 .map((note) => note.text),
-                           }) as ApiView,
-                     ),
-               }) as ApiSource,
-         );
+         .map((sourceObj) => {
+            const sourceName = sourceObj.as || sourceObj.name;
+            const annotations = (sourceObj as StructDef).annotation?.blockNotes
+               ?.filter((note) => note.at.url.includes(modelPath))
+               .map((note) => note.text);
+
+            // Parse #(filter) from ALL annotations (including imports)
+            // so filters defined on an imported source are honored by notebooks
+            const allAnnotations = (
+               sourceObj as StructDef
+            ).annotation?.blockNotes?.map((note) => note.text);
+            let filters: ApiFilter[] | undefined;
+            if (allAnnotations && allAnnotations.length > 0) {
+               try {
+                  const parsed = parseFilters(allAnnotations);
+                  if (parsed.length > 0) {
+                     filterMap.set(sourceName, parsed);
+                     const structFields = (sourceObj as StructDef).fields;
+                     filters = parsed.map((f) => {
+                        const field = structFields.find(
+                           (fd) => (fd.as || fd.name) === f.dimension,
+                        );
+                        return {
+                           name: f.name,
+                           dimension: f.dimension,
+                           type: f.type,
+                           implicit: f.implicit,
+                           required: f.required,
+                           dimensionType: field?.type as string | undefined,
+                        };
+                     });
+                  }
+               } catch (err) {
+                  logger.warn(
+                     `Failed to parse filter annotations on source "${sourceName}"`,
+                     { error: err },
+                  );
+               }
+            }
+
+            const views = (sourceObj as StructDef).fields
+               .filter((turtleObj) => turtleObj.type === "turtle")
+               .filter((turtleObj) =>
+                  // TODO(kjnesbit): Fix non-reduce views. Filter out
+                  // non-reduce views, i.e., indexes. Need to discuss with Will.
+                  (turtleObj as TurtleDef).pipeline
+                     .map((stage) => stage.type)
+                     .every((type) => type == "reduce"),
+               )
+               .map(
+                  (turtleObj) =>
+                     ({
+                        name: turtleObj.as || turtleObj.name,
+                        annotations: turtleObj?.annotation?.blockNotes
+                           ?.filter((note) => note.at.url.includes(modelPath))
+                           .map((note) => note.text),
+                     }) as ApiView,
+               );
+
+            return {
+               name: sourceName,
+               annotations,
+               views,
+               filters,
+            } as ApiSource;
+         });
+
+      return { sources, filterMap };
    }
 
    static async getModelMaterializer(
@@ -857,6 +993,7 @@ export class Model {
                      type: "code",
                      text: stmt.text,
                      runnable: runnable,
+                     modelMaterializer: localMM,
                      newSources,
                      queryInfo,
                   } as RunnableNotebookCell;