@malloy-publisher/server 0.0.202 → 0.0.204

package/dist/server.mjs

@@ -147328,6 +147328,8 @@ function internalErrorToHttpError(error) {
     return httpError(400, error.message);
   } else if (error instanceof FrozenConfigError) {
     return httpError(403, error.message);
+  } else if (error instanceof AccessDeniedError) {
+    return httpError(403, error.message);
   } else if (error instanceof EnvironmentNotFoundError) {
     return httpError(404, error.message);
   } else if (error instanceof PackageNotFoundError) {
@@ -147369,7 +147371,7 @@ function httpError(code, message) {
     }
   };
 }
-var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
+var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, ConnectionAuthError, ModelCompilationError, FrozenConfigError, AccessDeniedError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, QueryTimeoutError;
 var init_errors = __esm(() => {
   init_constants();
   NotImplementedError = class NotImplementedError extends Error {
@@ -147422,6 +147424,12 @@ var init_errors = __esm(() => {
       super(message);
     }
   };
+  AccessDeniedError = class AccessDeniedError extends Error {
+    constructor(message) {
+      super(message);
+      this.name = "AccessDeniedError";
+    }
+  };
   MaterializationNotFoundError = class MaterializationNotFoundError extends Error {
     constructor(message) {
       super(message);
@@ -199146,9 +199154,6 @@ function buildFetchOptions(options) {
   if (options.refreshTimestamp !== undefined) {
     out.refreshTimestamp = options.refreshTimestamp;
   }
-  if (options.modelAnnotation !== undefined) {
-    out.modelAnnotation = options.modelAnnotation;
-  }
   return out;
 }
 function adaptResult(result) {
@@ -221830,7 +221835,7 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
         ...azureDuckDBCache.values()
       ];
       const closeResults = await Promise.allSettled([
-        malloyConfig.releaseConnections(),
+        malloyConfig.shutdown("close"),
         ...wrapperPromises.map(async (promise) => {
           const connection = await promise;
           await connection.close();
@@ -229646,10 +229651,23 @@ function registerHealthEndpoints(app) {
 // src/service/environment_store.ts
 init_logger();
 
+// src/storage/StorageManager.ts
+import * as crypto3 from "crypto";
+
+// src/ducklake_version.ts
+var SUPPORTED_CATALOG_VERSIONS = [
+  "0.1",
+  "0.2",
+  "0.3-dev1",
+  "0.3"
+];
+function isCatalogVersionSupported(version) {
+  return SUPPORTED_CATALOG_VERSIONS.includes(version);
+}
+
 // src/storage/StorageManager.ts
 init_errors();
 init_logger();
-import * as crypto3 from "crypto";
 
 // src/storage/duckdb/DuckDBConnection.ts
 import duckdb from "duckdb";
@@ -230673,6 +230691,30 @@ function catalogNameForConfig(c) {
   const hash = crypto3.createHash("sha256").update(configKey(c)).digest("hex").slice(0, 8);
   return `manifest_lake_${hash}`;
 }
+async function readDuckLakeCatalogVersion(connection, catalogUrl, catalogName) {
+  if (!catalogUrl.startsWith("postgres:")) {
+    return;
+  }
+  const pgConnString = catalogUrl.slice("postgres:".length);
+  const tempDb = `${catalogName}_preflight`;
+  const escaped = escapeSQL2(pgConnString);
+  try {
+    await connection.run(`ATTACH '${escaped}' AS ${tempDb} (TYPE postgres, READ_ONLY);`);
+    const rows = await connection.all(`SELECT value FROM ${tempDb}.ducklake_metadata WHERE key = 'version' LIMIT 1;`);
+    const value = rows[0]?.value;
+    return typeof value === "string" ? value : undefined;
+  } catch (error) {
+    logger.warn("DuckLake catalog version preflight failed; falling back to ATTACH", {
+      catalogName,
+      error: redactPgSecrets(error instanceof Error ? error.message : String(error))
+    });
+    return;
+  } finally {
+    try {
+      await connection.run(`DETACH ${tempDb};`);
+    } catch {}
+  }
+}
 
 class StorageManager {
   connection = null;
@@ -230751,6 +230793,13 @@ class StorageManager {
     if (isCloudStorage) {
       await connection.run("INSTALL httpfs; LOAD httpfs;");
     }
+    if (isPostgres) {
+      const catalogVersion = await readDuckLakeCatalogVersion(connection, catalogUrl, catalogName);
+      if (catalogVersion && !isCatalogVersionSupported(catalogVersion)) {
+        const supportedMax = SUPPORTED_CATALOG_VERSIONS[SUPPORTED_CATALOG_VERSIONS.length - 1];
+        throw new ConnectionAuthError(`DuckLake catalog version ${catalogVersion} is newer than this Publisher's extension supports (max ${supportedMax}). Upgrade the Publisher image or downgrade the catalog.`);
+      }
+    }
     let attachCmd = `ATTACH 'ducklake:${escapedCatalogUrl}' AS ${catalogName}`;
     const attachOpts = [
       `DATA_PATH '${escapedDataPath}'`,
@@ -230861,9 +230910,9 @@ import {
 init_package_load_pool();
 var import_api4 = __toESM(require_src(), 1);
 import {
+  Annotations as Annotations2,
   API,
   FixedConnectionMap,
-  isSourceDef,
   MalloyConfig as MalloyConfig2,
   MalloyError as MalloyError2,
   modelDefToModelInfo,
@@ -231081,7 +231130,8 @@ function injectFilterRefinement(query, filterClause) {
   if (!filterClause) {
     return query;
   }
-  return `${query.trimEnd()} + {where: ${filterClause}}`;
+  return `${query.trimEnd()}
++ {where: ${filterClause}}`;
 }
 
 class FilterValidationError extends Error {
@@ -231120,6 +231170,149 @@ function tokenize(input) {
   return tokens;
 }
 
+// src/service/authorize.ts
+init_errors();
+var SOURCE_PREFIX = "#(authorize)";
+var FILE_PREFIX = "##(authorize)";
+function buildAuthorizeProbe(exprs) {
+  const selects = exprs.map((expr, i) => `__auth_${i} is (${expr})`).join(`
+      `);
+  return `run: duckdb.sql("SELECT 1 AS __authorize_probe_row") -> {
+    select:
+      ${selects}
+    limit: 1
+  }`;
+}
+function isProbeTrue(cell) {
+  return cell === true || cell === 1 || cell === "true";
+}
+async function evaluateAuthorize(executor, exprs, givens) {
+  for (const expr of exprs) {
+    try {
+      const result = await executor.loadQuery(buildAuthorizeProbe([expr])).run({ rowLimit: 1, givens });
+      const row = result?.data?.value?.[0];
+      if (row && isProbeTrue(row.__auth_0)) {
+        return true;
+      }
+    } catch {
+      continue;
+    }
+  }
+  return false;
+}
+async function validateAuthorizeProbes(compiler, sources) {
+  for (const source of sources) {
+    const exprs = source.authorize;
+    if (!exprs || exprs.length === 0)
+      continue;
+    try {
+      await compiler.loadQuery(buildAuthorizeProbe(exprs)).getPreparedQuery();
+    } catch (err) {
+      const detail = err instanceof Error ? err.message : String(err);
+      throw new ModelCompilationError({
+        message: `Invalid #(authorize) annotation on source "${source.name ?? "(unnamed)"}" [${exprs.join(" | ")}]: ${detail}`
+      });
+    }
+  }
+}
+function parseAuthorizeAnnotation(annotation) {
+  const trimmed2 = annotation.trim();
+  let body;
+  if (trimmed2.startsWith(FILE_PREFIX)) {
+    body = trimmed2.slice(FILE_PREFIX.length).trim();
+  } else if (trimmed2.startsWith(SOURCE_PREFIX)) {
+    body = trimmed2.slice(SOURCE_PREFIX.length).trim();
+  } else {
+    return null;
+  }
+  return unwrapQuotedExpression(body);
+}
+function collectAuthorizeExprs(annotations) {
+  const exprs = [];
+  for (const annotation of annotations) {
+    const expr = parseAuthorizeAnnotation(annotation);
+    if (expr !== null) {
+      exprs.push(expr);
+    }
+  }
+  return exprs;
+}
+function unwrapQuotedExpression(body) {
+  if (body.length < 2 || body[0] !== '"') {
+    throw new Error(`authorize annotation expression must be a double-quoted string, got: ${body || "(empty)"}`);
+  }
+  let expr = "";
+  let i = 1;
+  let closed = false;
+  for (;i < body.length; i++) {
+    const ch = body[i];
+    if (ch === "\\" && i + 1 < body.length) {
+      const next = body[i + 1];
+      if (next === '"' || next === "\\") {
+        expr += next;
+        i++;
+        continue;
+      }
+    }
+    if (ch === '"') {
+      closed = true;
+      i++;
+      break;
+    }
+    expr += ch;
+  }
+  if (!closed) {
+    throw new Error(`authorize annotation has mismatched quotes: ${body}`);
+  }
+  const rest = body.slice(i).trim();
+  if (rest.length > 0) {
+    throw new Error(`authorize annotation has unexpected content after the expression: ${rest}`);
+  }
+  if (expr.trim().length === 0) {
+    throw new Error("authorize annotation has an empty expression body");
+  }
+  return expr;
+}
+
+// src/service/annotations.ts
+import { Annotations } from "@malloydata/malloy";
+function isReservedRoute(route) {
+  return route === "" || !/[\p{L}\p{N}]/u.test(route);
+}
+function modelAnnotations(modelDef) {
+  const registry = modelDef.modelAnnotations ?? {};
+  const visited = new Set;
+  const order = [];
+  const visit = (id) => {
+    if (visited.has(id))
+      return;
+    visited.add(id);
+    const entry = registry[id];
+    if (!entry)
+      return;
+    for (const dep of entry.inheritsFrom)
+      visit(dep);
+    order.push(id);
+  };
+  visit(modelDef.modelID);
+  let folded;
+  for (const id of order) {
+    const own = registry[id].ownNotes;
+    if (!own.notes?.length && !own.blockNotes?.length)
+      continue;
+    folded = {
+      notes: own.notes,
+      blockNotes: own.blockNotes,
+      inherits: folded
+    };
+  }
+  return folded ?? {};
+}
+function annotationTexts(annote) {
+  const texts = new Annotations(annote).texts();
+  return texts.length > 0 ? texts : undefined;
+}
+
 // src/service/given.ts
 function malloyGivenToApi(given) {
   const type = given.type;
@@ -231127,10 +231320,89 @@ function malloyGivenToApi(given) {
   return {
     name: given.name,
     type: renderedType,
-    annotations: given.getTaglines(/^#\(/)
+    annotations: given.annotations.forRoute(undefined).filter((note) => !isReservedRoute(note.route)).map((note) => note.text),
+    default: given._internal?.defaultText
   };
 }
 
+// src/service/source_extraction.ts
+import {
+  isSourceDef
+} from "@malloydata/malloy";
+function extractSourcesFromModelDef(modelDef, givens, onParseError) {
+  const filterMap = new Map;
+  const authorizeMap = new Map;
+  const fileLevelAuthorize = collectAuthorizeExprs((modelAnnotations(modelDef).notes ?? []).map((note) => note.text));
+  const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
+    const struct = sourceObj;
+    const sourceName = struct.as || struct.name;
+    const annotations = annotationTexts(struct.annotations);
+    const collected = [];
+    let cur = struct.annotations;
+    while (cur) {
+      if (cur.blockNotes) {
+        collected.push(cur.blockNotes.map((note) => note.text));
+      }
+      cur = cur.inherits;
+    }
+    const allAnnotations = collected.reverse().flat();
+    let filters;
+    if (allAnnotations.length > 0) {
+      try {
+        const parsed = parseFilters(allAnnotations);
+        if (parsed.length > 0) {
+          filterMap.set(sourceName, parsed);
+          const fields = struct.fields;
+          filters = parsed.map((f) => {
+            const field = fields.find((fd) => (fd.as || fd.name) === f.dimension);
+            return {
+              name: f.name,
+              dimension: f.dimension,
+              type: f.type,
+              implicit: f.implicit,
+              required: f.required,
+              dimensionType: field?.type
+            };
+          });
+        }
+      } catch (err) {
+        onParseError?.(sourceName, err);
+      }
+    }
+    const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
+    const effective = [
+      ...fileLevelAuthorize,
+      ...collectAuthorizeExprs(ownNotes)
+    ];
+    let authorize;
+    if (effective.length > 0) {
+      authorizeMap.set(sourceName, effective);
+      authorize = effective;
+    }
+    const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
+      name: turtle.as || turtle.name,
+      annotations: annotationTexts(turtle.annotations)
+    }));
+    return {
+      name: sourceName,
+      annotations,
+      views,
+      filters,
+      givens,
+      authorize
+    };
+  });
+  return { sources, filterMap, authorizeMap };
+}
+function extractQueriesFromModelDef(modelDef) {
+  const isNamedQuery = (obj) => obj.type === "query";
+  return Object.values(modelDef.contents).filter(isNamedQuery).map((queryObj) => ({
+    name: queryObj.as || queryObj.name,
+    sourceName: typeof queryObj.structRef === "string" ? queryObj.structRef : undefined,
+    annotations: annotationTexts(queryObj.annotations)
+  }));
+}
+
 // src/service/model_limits.ts
 init_errors();
 function resolveModelQueryRowLimit(userLimit, { defaultLimit, maxRows }) {
@@ -231168,6 +231440,7 @@ class Model {
   compilationError;
   filterMap;
   givens;
+  fileLevelAuthorize = [];
   meter = import_api4.metrics.getMeter("publisher");
   queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
     description: "How long it takes to execute a Malloy model query",
@@ -231187,6 +231460,11 @@ class Model {
     this.compilationError = compilationError;
     this.filterMap = filterMap ?? new Map;
     this.givens = givens;
+    try {
+      this.fileLevelAuthorize = this.modelDef ? collectAuthorizeExprs((modelAnnotations(this.modelDef).notes ?? []).map((note) => note.text)) : [];
+    } catch {
+      this.fileLevelAuthorize = [];
+    }
     this.modelInfo = modelInfo ?? (this.modelDef ? modelDefToModelInfo(this.modelDef) : undefined);
     if (this.filterMap.size > 0) {
       logger.warn(`Model "${packageName}/${modelPath}" uses deprecated #(filter) annotations. Migrate to given: — see https://github.com/malloydata/publisher/blob/main/docs/givens.md`, {
@@ -231199,12 +231477,78 @@ class Model {
   getFilters(sourceName) {
     return this.filterMap.get(sourceName) ?? [];
   }
+  getAuthorize(sourceName) {
+    return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
+  }
+  effectiveAuthorizeFor(sourceName) {
+    if (sourceName && this.sources?.some((s) => s.name === sourceName)) {
+      return this.getAuthorize(sourceName);
+    }
+    return this.fileLevelAuthorize;
+  }
+  async assertAuthorized(sourceName, givens) {
+    const exprs = this.effectiveAuthorizeFor(sourceName);
+    if (exprs.length === 0)
+      return;
+    const label = sourceName ?? "(query)";
+    const deny = () => {
+      throw new AccessDeniedError(`Access denied for source "${label}".`);
+    };
+    if (!this.modelMaterializer)
+      deny();
+    let passed = false;
+    try {
+      passed = await evaluateAuthorize(this.modelMaterializer, exprs, givens);
+    } catch (err) {
+      logger.debug("Authorize probe failed; denying", {
+        sourceName: label,
+        modelPath: this.modelPath,
+        error: err instanceof Error ? err.message : String(err)
+      });
+      deny();
+    }
+    if (!passed)
+      deny();
+  }
+  async resolveAuthorizeSourceFromRunnable(runnable) {
+    try {
+      const prepared = await runnable.getPreparedQuery();
+      const structRef = prepared._query?.structRef;
+      if (typeof structRef === "string")
+        return structRef;
+      if (structRef && typeof structRef === "object") {
+        const s = structRef;
+        return s.as || s.name;
+      }
+    } catch {}
+    return;
+  }
   extractSourceName(query) {
     if (!query)
       return;
-    const runMatch = query.match(/run\s*:\s*(\w+)\s*->/);
-    const arrowMatch = query.match(/^\s*(\w+)\s*->/m);
-    return runMatch?.[1] ?? arrowMatch?.[1];
+    const runMatch = query.match(/run\s*:\s*(?:`([^`]+)`|(\w+))\s*->/);
+    const arrowMatch = query.match(/^\s*(?:`([^`]+)`|(\w+))\s*->/m);
+    return runMatch?.[1] ?? runMatch?.[2] ?? arrowMatch?.[1] ?? arrowMatch?.[2];
+  }
+  resolveFilterSource(query) {
+    const target = this.extractSourceName(query);
+    if (!target || !query)
+      return;
+    const aliasOf = new Map;
+    const declRe = /source\s*:\s*(\w+)\s+is\s+(\w+)/g;
+    let match;
+    while ((match = declRe.exec(query)) !== null) {
+      aliasOf.set(match[1], match[2]);
+    }
+    let current = target;
+    const seen = new Set;
+    while (current && !seen.has(current)) {
+      if (this.filterMap.has(current))
+        return current;
+      seen.add(current);
+      current = aliasOf.get(current);
+    }
+    return;
   }
   static async create(packageName, packagePath, modelPath, malloyConfig, options) {
     const { runtime, modelURL, importBaseURL, dataStyles, modelType } = await Model.getModelRuntime(packagePath, modelPath, malloyConfig, options);
@@ -231221,10 +231565,11 @@ class Model {
         modelDef = compiledModel._modelDef;
         const malloyGivens = Array.from(compiledModel.givens.values());
         givens = malloyGivens.length > 0 ? malloyGivens.map(malloyGivenToApi) : undefined;
-        const sourceResult = Model.getSources(modelPath, modelDef, givens);
+        const sourceResult = Model.getSources(modelDef, givens);
         sources = sourceResult.sources;
         filterMap = sourceResult.filterMap;
-        queries = Model.getQueries(modelPath, modelDef);
+        queries = Model.getQueries(modelDef);
+        await validateAuthorizeProbes(modelMaterializer, sources ?? []);
         const imports = modelDef.imports || [];
         const importedSourceNames = new Set;
         for (const importLocation of imports) {
@@ -231340,6 +231685,10 @@ class Model {
     let runnable;
     if (!this.modelMaterializer || !this.modelDef || !this.modelInfo)
       throw new BadRequestError("Model has no queryable entities.");
+    const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || this.extractSourceName(query);
+    if (earlySource) {
+      await this.assertAuthorized(earlySource, givens ?? {});
+    }
     try {
       let queryString;
       if (!sourceName && !queryName && query) {
@@ -231360,8 +231709,9 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
         });
         throw new BadRequestError("Invalid query request. (Query AND !sourceName) OR (queryName AND sourceName) must be defined.");
       }
+      const isAdHocQuery = !sourceName && !queryName && !!query;
       if (!bypassFilters) {
-        const effectiveSource = sourceName ?? this.extractSourceName(query);
+        const effectiveSource = isAdHocQuery ? this.resolveFilterSource(query) : sourceName;
         if (effectiveSource) {
           const filters = this.getFilters(effectiveSource);
           if (filters.length > 0) {
@@ -231370,7 +231720,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
           }
         }
       }
-      runnable = this.modelMaterializer.loadQuery(queryString);
+      runnable = this.modelMaterializer.loadRestrictedQuery(queryString);
     } catch (error) {
       if (error instanceof BadRequestError) {
         throw error;
@@ -231393,6 +231743,10 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       });
       throw new BadRequestError(`Invalid query: ${errorMessage}`);
     }
+    const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
+    if (!(compiledSource && compiledSource === earlySource)) {
+      await this.assertAuthorized(compiledSource, givens ?? {});
+    }
     const maxRows = getMaxQueryRows();
     const maxBytes = getMaxResponseBytes();
     const rowLimit = resolveModelQueryRowLimit((await runnable.getPreparedResult({ givens })).resultExplore.limit, { defaultLimit: getDefaultQueryRowLimit(), maxRows });
@@ -231461,25 +231815,19 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       givens: this.givens
     };
   }
+  serializeNewSources(newSources) {
+    return newSources?.map((source) => JSON.stringify(this.givens && this.givens.length > 0 ? { ...source, givens: this.givens } : source));
+  }
   async getNotebookModel() {
     const notebookCells = this.runnableNotebookCells.map((cell) => {
       return {
         type: cell.type,
         text: cell.text,
-        newSources: cell.newSources?.map((source) => JSON.stringify(source)),
+        newSources: this.serializeNewSources(cell.newSources),
         queryInfo: cell.queryInfo ? JSON.stringify(cell.queryInfo) : undefined
       };
     });
-    const allAnnotations = [];
-    if (this.modelDef) {
-      let currentAnnotation = this.modelDef.annotation;
-      while (currentAnnotation) {
-        if (currentAnnotation.notes) {
-          allAnnotations.push(...currentAnnotation.notes.map((note) => note.text));
-        }
-        currentAnnotation = currentAnnotation.inherits;
-      }
-    }
+    const allAnnotations = this.modelDef ? new Annotations2(modelAnnotations(this.modelDef)).texts() : [];
     return {
       type: "notebook",
       packageName: this.packageName,
@@ -231509,6 +231857,10 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
         text: cell.text
       };
     }
+    if (cell.runnable) {
+      const authorizeSource = await this.resolveAuthorizeSourceFromRunnable(cell.runnable);
+      await this.assertAuthorized(authorizeSource, givens ?? {});
+    }
     let queryName = undefined;
     let queryResult = undefined;
     if (cell.runnable) {
@@ -231571,7 +231923,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       text: cell.text,
       queryName,
       result: queryResult,
-      newSources: cell.newSources?.map((source) => JSON.stringify(source))
+      newSources: this.serializeNewSources(cell.newSources)
     };
   }
   static async getModelRuntime(packagePath, modelPath, malloyConfig, options) {
@@ -231611,63 +231963,11 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
     malloyConfig.wrapConnections(() => new FixedConnectionMap(input, "duckdb"));
     return malloyConfig;
   }
-  static getQueries(modelPath, modelDef) {
-    const isNamedQuery = (object) => object.type === "query";
-    return Object.values(modelDef.contents).filter(isNamedQuery).map((queryObj) => ({
-      name: queryObj.as || queryObj.name,
-      sourceName: typeof queryObj.structRef === "string" ? queryObj.structRef : undefined,
-      annotations: queryObj?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
-    }));
+  static getQueries(modelDef) {
+    return extractQueriesFromModelDef(modelDef);
   }
-  static getSources(modelPath, modelDef, givens) {
-    const filterMap = new Map;
-    const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef(obj)).map((sourceObj) => {
-      const sourceName = sourceObj.as || sourceObj.name;
-      const annotations = sourceObj.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text);
-      const collectedAnnotations = [];
-      let curAnnotation = sourceObj.annotation;
-      while (curAnnotation) {
-        if (curAnnotation.blockNotes) {
-          collectedAnnotations.push(curAnnotation.blockNotes.map((note) => note.text));
-        }
-        curAnnotation = curAnnotation.inherits;
-      }
-      const allAnnotations = collectedAnnotations.reverse().flat();
-      let filters;
-      if (allAnnotations.length > 0) {
-        try {
-          const parsed = parseFilters(allAnnotations);
-          if (parsed.length > 0) {
-            filterMap.set(sourceName, parsed);
-            const structFields = sourceObj.fields;
-            filters = parsed.map((f) => {
-              const field = structFields.find((fd) => (fd.as || fd.name) === f.dimension);
-              return {
-                name: f.name,
-                dimension: f.dimension,
-                type: f.type,
-                implicit: f.implicit,
-                required: f.required,
-                dimensionType: field?.type
-              };
-            });
-          }
-        } catch (err) {
-          logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err });
-        }
-      }
-      const views = sourceObj.fields.filter((turtleObj) => turtleObj.type === "turtle").filter((turtleObj) => turtleObj.pipeline.map((stage) => stage.type).every((type) => type == "reduce")).map((turtleObj) => ({
-        name: turtleObj.as || turtleObj.name,
-        annotations: turtleObj?.annotation?.blockNotes?.filter((note) => note.at.url.includes(modelPath)).map((note) => note.text)
-      }));
-      return {
-        name: sourceName,
-        annotations,
-        views,
-        filters,
-        givens
-      };
-    });
+  static getSources(modelDef, givens) {
+    const { sources, filterMap } = extractSourcesFromModelDef(modelDef, givens, (sourceName, err) => logger.warn(`Failed to parse filter annotations on source "${sourceName}"`, { error: err }));
     return { sources, filterMap };
   }
   static async getModelMaterializer(runtime, importBaseURL, modelURL, modelPath) {
@@ -232470,7 +232770,7 @@ ${source}` : source;
       const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
       const _package = await Package.create(this.environmentName, packageName, packagePath, () => this.malloyConfig.malloyConfig);
       if (existingPackage !== undefined && reload) {
-        this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().releaseConnections());
+        this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
       }
       this.packages.set(packageName, _package);
       this.setPackageStatus(packageName, "serving" /* SERVING */);
@@ -232592,7 +232892,7 @@ ${source}` : source;
       this.packages.set(packageName, newPackage);
       this.setPackageStatus(packageName, "serving" /* SERVING */);
       if (oldPackage) {
-        this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().releaseConnections());
+        this.retireConnectionGeneration(`package ${packageName}`, () => oldPackage.getMalloyConfig().shutdown("close"));
       }
       if (retiredPath) {
         const pathToClean = retiredPath;
@@ -232708,7 +233008,7 @@ ${source}` : source;
       } else if (packageStatus?.status === "serving" /* SERVING */) {
         this.setPackageStatus(packageName, "unloading" /* UNLOADING */);
       }
-      this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().releaseConnections());
+      this.retireConnectionGeneration(`package ${packageName}`, () => _package.getMalloyConfig().shutdown("close"));
       const canonicalPath = safeJoinUnderRoot(this.environmentPath, packageName);
       const retiredPath = this.allocateRetiredPath(packageName);
       let renamed = false;
@@ -232761,7 +233061,7 @@ ${source}` : source;
     }
   }
   async closeAllConnections() {
-    const packageReleases = await Promise.allSettled(Array.from(this.packages.values(), (pkg) => pkg.getMalloyConfig().releaseConnections()));
+    const packageReleases = await Promise.allSettled(Array.from(this.packages.values(), (pkg) => pkg.getMalloyConfig().shutdown("close")));
     for (const result of packageReleases) {
       if (result.status === "rejected") {
         logger.error(`Error closing package connections for environment ${this.environmentName}`, { error: result.reason });
@@ -233188,12 +233488,14 @@ class EnvironmentStore {
     return Promise.all(Array.from(this.environments.values()).map((environment) => environment.serialize()));
   }
   async getStatus() {
+    const baseState = getOperationalState();
+    const operationalState2 = baseState !== "draining" && (this.memoryGovernor?.isBackpressured() ?? false) ? "throttled" : baseState;
     const status = {
       timestamp: Date.now(),
       environments: [],
       initialized: this.isInitialized,
       frozenConfig: isPublisherConfigFrozen(this.serverRootPath),
-      operationalState: getOperationalState()
+      operationalState: operationalState2
     };
     const environments = await this.listEnvironments(true);
     await Promise.all(environments.map(async (environment) => {
@@ -238655,41 +238957,6 @@ import { Manifest } from "@malloydata/malloy";
 
 // src/service/materialized_table_gc.ts
 init_logger();
-import {
-  DatabricksDialect,
-  DuckDBDialect,
-  MySQLDialect,
-  PostgresDialect,
-  SnowflakeDialect,
-  StandardSQLDialect,
-  TrinoDialect
-} from "@malloydata/malloy";
-
-// src/service/quoting.ts
-function quoteTablePath(path10, dialect) {
-  return path10.split(".").map((seg) => dialect.quoteTablePath(seg)).join(".");
-}
-function splitTablePath(tableName) {
-  const lastDot = tableName.lastIndexOf(".");
-  if (lastDot >= 0) {
-    return {
-      schemaPrefix: tableName.substring(0, lastDot + 1),
-      bareName: tableName.substring(lastDot + 1)
-    };
-  }
-  return { schemaPrefix: "", bareName: tableName };
-}
-
-// src/service/materialized_table_gc.ts
-var DIALECTS = Object.freeze({
-  duckdb: new DuckDBDialect,
-  standardsql: new StandardSQLDialect,
-  trino: new TrinoDialect,
-  postgres: new PostgresDialect,
-  snowflake: new SnowflakeDialect,
-  mysql: new MySQLDialect,
-  databricks: new DatabricksDialect
-});
 function liveTableKey(connectionName, tableName) {
   return `${connectionName}::${tableName}`;
 }
@@ -238735,17 +239002,6 @@ async function processOneEntry(entry, ctx, liveTables) {
       }
     };
   }
-  const dialect = DIALECTS[connection.dialectName];
-  if (!dialect) {
-    return {
-      error: {
-        buildId: entry.buildId,
-        tableName: entry.tableName,
-        connectionName: entry.connectionName,
-        error: `No dialect registered for '${connection.dialectName}'`
-      }
-    };
-  }
   if (ctx.dryRun) {
     return {
       dropped: {
@@ -238757,7 +239013,6 @@ async function processOneEntry(entry, ctx, liveTables) {
       }
     };
   }
-  const quoted = (p) => quoteTablePath(p, dialect);
   try {
     await ctx.manifestService.deleteEntry(ctx.environmentId, entry.id);
   } catch (err) {
@@ -238784,7 +239039,7 @@ async function processOneEntry(entry, ctx, liveTables) {
     });
   } else {
     try {
-      await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(entry.tableName)}`);
+      await connection.runSQL(`DROP TABLE IF EXISTS ${entry.tableName}`);
     } catch (err) {
       logger.warn("GC: deleted manifest row but failed to drop materialized table (orphaned)", {
         tableName: entry.tableName,
@@ -238794,7 +239049,7 @@ async function processOneEntry(entry, ctx, liveTables) {
     }
   }
   try {
-    await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(stagingTableName)}`);
+    await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
   } catch (err) {
     logger.warn("GC: failed to drop staging table (best-effort)", {
       stagingTableName,
@@ -238826,6 +239081,18 @@ async function dropManifestEntries(entries, ctx) {
   return { dropped, errors: errors2 };
 }
 
+// src/service/quoting.ts
+function splitTablePath(tableName) {
+  const lastDot = tableName.lastIndexOf(".");
+  if (lastDot >= 0) {
+    return {
+      schemaPrefix: tableName.substring(0, lastDot + 1),
+      bareName: tableName.substring(lastDot + 1)
+    };
+  }
+  return { schemaPrefix: "", bareName: tableName };
+}
+
 // src/service/materialization_service.ts
 var STAGING_BUILD_ID_LEN = 12;
 function stagingSuffix(buildId) {
@@ -238851,9 +239118,9 @@ async function resolvePackageConnections(pkg, names) {
 function manifestTableKey(connectionName, tableName) {
   return `${connectionName}::${tableName}`;
 }
-async function tablePhysicallyExists(connection, quotedTableName) {
+async function tablePhysicallyExists(connection, tableName) {
   try {
-    await connection.runSQL(`SELECT 1 FROM ${quotedTableName} WHERE 1=0`);
+    await connection.runSQL(`SELECT 1 FROM ${tableName} WHERE 1=0`);
     return true;
   } catch {
     return false;
@@ -239108,7 +239375,7 @@ class MaterializationService {
         importBaseURL
       });
       const malloyModel = await modelMaterializer.getModel();
-      const modelTag = malloyModel.tagParse({ prefix: /^##! / }).tag;
+      const modelTag = malloyModel.annotations.parseAsTag("!").tag;
       if (!modelTag.has("experimental", "persistence")) {
         logger.debug("Model has no ##! experimental.persistence tag, skipping", { modelPath });
         continue;
@@ -239138,7 +239405,7 @@ class MaterializationService {
     });
     const tableOwners = new Map;
     for (const [sourceID, source] of Object.entries(allSources)) {
-      const tableName = source.tagParse({ prefix: /^#@ / }).tag.text("name") || source.name;
+      const tableName = source.annotations.parseAsTag("@").tag.text("name") || source.name;
       const key = `${source.connectionName}::${tableName}`;
       const existing = tableOwners.get(key);
       if (existing) {
@@ -239177,14 +239444,12 @@ class MaterializationService {
       connectionDigests
     });
     const connectionName = persistSource.connectionName;
-    const tableName = persistSource.tagParse({ prefix: /^#@ / }).tag.text("name") || persistSource.name;
-    const { schemaPrefix, bareName } = splitTablePath(tableName);
-    const stagingTableName = `${schemaPrefix}${bareName}${stagingSuffix(buildId)}`;
-    const dialect = persistSource.dialect;
-    const quoted = (p) => quoteTablePath(p, dialect);
+    const tableName = persistSource.annotations.parseAsTag("@").tag.text("name") || persistSource.name;
+    const { bareName } = splitTablePath(tableName);
+    const stagingTableName = `${tableName}${stagingSuffix(buildId)}`;
     const tableKey = manifestTableKey(connectionName, tableName);
     if (!knownMaterializedTables.has(tableKey)) {
-      if (await tablePhysicallyExists(connection, quoted(tableName))) {
+      if (await tablePhysicallyExists(connection, tableName)) {
         throw new BadRequestError(`Refusing to materialize source '${persistSource.name}': ` + `target table '${tableName}' already exists on connection ` + `'${connectionName}' but was not created by a previous ` + `materialization build. Use '#@ persist name=...' to ` + `choose a different table name, or drop the existing ` + `table manually if it is no longer needed.`);
       }
     }
@@ -239193,14 +239458,14 @@ class MaterializationService {
       connectionName
     });
     const startTime = performance.now();
-    await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(stagingTableName)}`);
+    await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
     try {
-      await connection.runSQL(`CREATE TABLE ${quoted(stagingTableName)} AS (${buildSQL})`);
-      await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(tableName)}`);
-      await connection.runSQL(`ALTER TABLE ${quoted(stagingTableName)} RENAME TO ${dialect.quoteTablePath(bareName)}`);
+      await connection.runSQL(`CREATE TABLE ${stagingTableName} AS (${buildSQL})`);
+      await connection.runSQL(`DROP TABLE IF EXISTS ${tableName}`);
+      await connection.runSQL(`ALTER TABLE ${stagingTableName} RENAME TO ${bareName}`);
     } catch (err) {
       try {
-        await connection.runSQL(`DROP TABLE IF EXISTS ${quoted(stagingTableName)}`);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${stagingTableName}`);
       } catch (cleanupErr) {
         logger.warn("Build: failed to clean up staging table after a failed rebuild; physical leak", {
           stagingTableName,