@malloy-publisher/server 0.9.6 → 0.9.7

package/dist/server.mjs

@@ -205143,7 +205143,7 @@ function installRecordPath(environmentPath, packageName) {
 function normalizeModelPath(p) {
   return p.replace(/\\/g, "/").replace(/^\.\//, "");
 }
-var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", PACKAGE_INSTALL_RECORDS_DIR = ".install-records", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", INDEX_MODEL_NAME = "index.malloy", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, DEFAULT_POSTGRES_POOL_MAX = 5, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
+var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", PACKAGE_INSTALL_RECORDS_DIR = ".install-records", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", GIVENS_DOCS_URL = "https://github.com/malloydata/publisher/blob/main/docs/givens.md", INDEX_MODEL_NAME = "index.malloy", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, DEFAULT_POSTGRES_POOL_MAX = 5, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
 var init_constants = __esm(() => {
   TEMP_DIR_PATH = os.tmpdir();
 });
@@ -205174,6 +205174,59 @@ function filesystemAccessFailure(error2) {
   }
   return;
 }
+function isMysqlConfigFault(code) {
+  return typeof code === "string" && (MYSQL_CONFIG_CODES.has(code) || code.startsWith("HANDSHAKE_"));
+}
+function isConnectionFailure(error2) {
+  let current = error2;
+  for (let depth = 0;depth < 8 && current instanceof Error; depth++) {
+    const { code, fatal } = current;
+    if (typeof code === "string") {
+      if (NODE_CONNECTION_CODES.has(code))
+        return true;
+      if (POSTGRES_CONNECTION_SQLSTATE.test(code))
+        return true;
+    }
+    if (fatal === true && !isMysqlConfigFault(code))
+      return true;
+    const message = current.message.trim();
+    if (CODELESS_CONNECTION_MESSAGES.some((re) => re.test(message))) {
+      return true;
+    }
+    current = current.cause;
+  }
+  return false;
+}
+function isCredentialRejection(error2) {
+  let current = error2;
+  for (let depth = 0;depth < 8 && current instanceof Error; depth++) {
+    const { code } = current;
+    if (typeof code === "string" || typeof code === "number") {
+      const value = String(code);
+      if (POSTGRES_AUTH_SQLSTATE.test(value))
+        return true;
+      if (MYSQL_AUTH_CODES.has(value))
+        return true;
+      if (SNOWFLAKE_AUTH_CODES.has(value))
+        return true;
+    }
+    if (current.name === "BigQueryAuthenticationError")
+      return true;
+    current = current.cause;
+  }
+  return false;
+}
+function databaseAccessFailure(error2) {
+  if (error2 instanceof ConnectionAuthError || error2 instanceof ConnectionFailedError || error2 instanceof UnconfiguredConnectionError) {
+    return error2;
+  }
+  const message = error2 instanceof Error ? error2.message : String(error2);
+  if (isCredentialRejection(error2))
+    return new ConnectionAuthError(message);
+  if (isConnectionFailure(error2))
+    return new ConnectionFailedError(message);
+  return;
+}
 function describeFilesystemAccessFailure(errno) {
   const code = errno.code ?? "";
   const target = errno.path ? ` '${errno.path}'` : "";
@@ -205243,7 +205296,13 @@ function internalErrorToHttpError(error2, { log = true } = {}) {
   } else if (error2 instanceof DestinationNotFoundError) {
     return httpError(422, error2.message);
   } else if (error2 instanceof ConnectionAuthError) {
-    return httpError(422, error2.message);
+    logInternal("Connection credentials rejected", error2, "warn");
+    return httpError(424, GENERIC_AUTH_MESSAGE, "CONNECTION_AUTH_FAILED");
+  } else if (error2 instanceof ConnectionFailedError) {
+    logInternal("Database unreachable", error2, "warn");
+    return httpError(502, GENERIC_UNREACHABLE_MESSAGE, "CONNECTION_FAILED");
+  } else if (error2 instanceof UnconfiguredConnectionError) {
+    return httpError(424, error2.message, "CONNECTION_NOT_FOUND");
   } else if (error2 instanceof UnsupportedCatalogFormatError) {
     return httpError(422, error2.message);
   } else if (error2 instanceof MaterializationEligibilityError) {
@@ -205300,7 +205359,7 @@ function toQueryTextProblem(problem) {
     ...problem.at ? { at: { range: problem.at.range } } : {}
   };
 }
-var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, RenderTagRefusedError, UnparseableTextError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, ConnectionPoolExhaustedError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, ChainedUpstreamMissingError, ChainedShapeNotCarriedError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, WriteVerifyError, InvalidStateTransitionError, ServiceUnavailableError, PackageAdmissionRefusedError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
+var GENERIC_INTERNAL_MESSAGE = "Internal server error.", GENERIC_UPSTREAM_MESSAGE = "Upstream connection error.", GENERIC_UNREACHABLE_MESSAGE = "The database connection is down: the database could not be reached, so the query did not run.", GENERIC_AUTH_MESSAGE = "The database rejected the connection's credentials. Check the connection's user, password, key or token.", MAX_LOGGED_DETAIL_CHARS = 2000, FILESYSTEM_ACCESS_DESCRIPTIONS, NODE_CONNECTION_CODES, POSTGRES_CONNECTION_SQLSTATE, CODELESS_CONNECTION_MESSAGES, MYSQL_CONFIG_CODES, POSTGRES_AUTH_SQLSTATE, MYSQL_AUTH_CODES, SNOWFLAKE_AUTH_CODES, NotImplementedError, BadRequestError, InvalidArgumentError, CompileRefusedError, RenderTagRefusedError, UnparseableTextError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, DashboardNotFoundError, ConnectionNotFoundError, TableNotFoundError, ConnectionError, ConnectionFailedError, UnconfiguredConnectionError, ConnectionPoolExhaustedError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, PackageManifestError, MaterializationEligibilityError, ChainedUpstreamMissingError, ChainedShapeNotCarriedError, PublisherConfigError, ServerConfigurationError, FrozenConfigError, AccessDeniedError, QueryCompileError, NotQueryableError, OffSurfaceError, MaterializationNotFoundError, MaterializationConflictError, WriteConflictError, WriteRolledBackError, WriteVerifyError, InvalidStateTransitionError, ServiceUnavailableError, PackageAdmissionRefusedError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
 var init_errors = __esm(() => {
   init_constants();
   init_logger();
@@ -205309,6 +205368,35 @@ var init_errors = __esm(() => {
     EPERM: "operation not permitted",
     EROFS: "read-only file system"
   };
+  NODE_CONNECTION_CODES = new Set([
+    "ECONNREFUSED",
+    "ECONNRESET",
+    "ECONNABORTED",
+    "EPIPE",
+    "ETIMEDOUT",
+    "EAI_AGAIN",
+    "EHOSTUNREACH",
+    "ENETUNREACH"
+  ]);
+  POSTGRES_CONNECTION_SQLSTATE = /^(08[0-9A-Z]{3}|57P0[123])$/;
+  CODELESS_CONNECTION_MESSAGES = [
+    /^Connection terminated unexpectedly$/,
+    /^(Error: )?Can't add new command when connection is in closed state$/,
+    /^(Error: )?Connection lost: The server closed the connection\.$/,
+    new RegExp(`^(Error: )?(connect|read|write|getaddrinfo) (${[
+      ...NODE_CONNECTION_CODES
+    ].join("|")})\\b`)
+  ];
+  MYSQL_CONFIG_CODES = new Set([
+    "ER_ACCESS_DENIED_ERROR",
+    "ER_BAD_DB_ERROR",
+    "ER_NOT_SUPPORTED_AUTH_MODE",
+    "AUTH_SWITCH_PLUGIN_ERROR",
+    "MYSQL_CLEAR_PASSWORD_NOT_ENABLED"
+  ]);
+  POSTGRES_AUTH_SQLSTATE = /^28[0-9A-Z]{3}$/;
+  MYSQL_AUTH_CODES = new Set(["ER_ACCESS_DENIED_ERROR"]);
+  SNOWFLAKE_AUTH_CODES = new Set(["390100", "390144", "390195", "390318"]);
   NotImplementedError = class NotImplementedError extends Error {
     constructor(message) {
       super(message);
@@ -205333,8 +205421,10 @@ var init_errors = __esm(() => {
     }
   };
   EnvironmentNotFoundError = class EnvironmentNotFoundError extends Error {
-    constructor(message) {
+    lookup;
+    constructor(message, lookup) {
       super(message);
+      this.lookup = lookup;
     }
   };
   PackageNotFoundError = class PackageNotFoundError extends Error {
@@ -205369,6 +205459,13 @@ var init_errors = __esm(() => {
       this.callerSafe = options?.callerSafe ?? false;
     }
   };
+  ConnectionFailedError = class ConnectionFailedError extends ConnectionError {
+  };
+  UnconfiguredConnectionError = class UnconfiguredConnectionError extends Error {
+    constructor(name, options) {
+      super(`No connection named "${name}" found in config`, options);
+    }
+  };
   ConnectionPoolExhaustedError = class ConnectionPoolExhaustedError extends ConnectionError {
     constructor(message) {
       super(message, { callerSafe: true });
@@ -206198,6 +206295,22 @@ function processConfigValue(value) {
   }
   return value;
 }
+function sanitizeMcpConfig(raw) {
+  if (raw === undefined)
+    return;
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) {
+    logger.warn(`Invalid "mcp" in ${PUBLISHER_CONFIG_NAME}: expected an object. Ignoring.`);
+    return;
+  }
+  const mcp = {};
+  const value = raw.includeHiddenFilesAndSources;
+  if (typeof value === "boolean") {
+    mcp.includeHiddenFilesAndSources = value;
+  } else if (value !== undefined) {
+    logger.warn(`Invalid "mcp.includeHiddenFilesAndSources" in ${PUBLISHER_CONFIG_NAME}: expected a boolean (got ${JSON.stringify(value)}). Ignoring field.`);
+  }
+  return Object.keys(mcp).length > 0 ? mcp : undefined;
+}
 function sanitizeTheme(raw, context) {
   if (raw === undefined || raw === null)
     return;
@@ -206550,10 +206663,12 @@ var BUNDLED_DEFAULT_CONFIG_PATH, PER_MODE_COLOR_KEYS, DEFAULT_HIGH_WATER_FRACTIO
     frozenConfig = Boolean(processedConfig.frozenConfig);
   }
   const instanceTheme = sanitizeTheme(processedConfig && typeof processedConfig === "object" && "theme" in processedConfig ? processedConfig.theme : undefined, "publisher.config.json");
+  const mcp = sanitizeMcpConfig(processedConfig && typeof processedConfig === "object" && "mcp" in processedConfig ? processedConfig.mcp : undefined);
   const retrieval = parseRetrievalConfig(processedConfig && typeof processedConfig === "object" ? processedConfig.retrieval : undefined);
   return {
     frozenConfig,
     ...instanceTheme ? { theme: instanceTheme } : {},
+    ...mcp ? { mcp } : {},
     ...retrieval ? { retrieval } : {},
     environments
   };
@@ -206565,6 +206680,13 @@ var BUNDLED_DEFAULT_CONFIG_PATH, PER_MODE_COLOR_KEYS, DEFAULT_HIGH_WATER_FRACTIO
     logger.error(`Error checking if ${PUBLISHER_CONFIG_NAME} is frozen. Defaulting to false.`, { error: error2 instanceof Error ? error2.message : String(error2) });
     return false;
   }
+}, isMcpIncludeHiddenFilesAndSources = (serverRoot) => {
+  try {
+    return getPublisherConfig(serverRoot).mcp?.includeHiddenFilesAndSources === true;
+  } catch (error2) {
+    logger.error(`Error reading "mcp" from ${PUBLISHER_CONFIG_NAME}. Defaulting to off.`, { error: error2 instanceof Error ? error2.message : String(error2) });
+    return false;
+  }
 }, convertConnectionsToApiConnections = (connections) => {
   if (!Array.isArray(connections)) {
     return [];
@@ -269128,11 +269250,6 @@ var require_utils93 = __commonJS((exports, module) => {
   };
 });
 
-// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
-var require_sshcrypto = __commonJS((exports, module) => {
-  module.exports = __require("./sshcrypto-cqb52edz.node");
-});
-
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
 var require_poly1305 = __commonJS((exports, module) => {
   var __dirname = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto", __filename = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto/poly1305.js";
@@ -269619,7 +269736,7 @@ var require_crypto = __commonJS((exports, module) => {
   var ChaChaPolyDecipher;
   var GenericDecipher;
   try {
-    binding = require_sshcrypto();
+    binding = (()=>{throw new Error("Cannot require module "+"./crypto/build/Release/sshcrypto.node");})();
     ({
       AESGCMCipher,
       ChaChaPolyCipher,
@@ -284149,7 +284266,6 @@ function assembleEnvironmentConnections(connections = [], environmentPath = "")
     metadata.set(connection.name, {
       apiConnection,
       attachedDatabases,
-      setupSQL: connection.duckdbConnection?.setupSQL,
       hasAzureAttachment: attachedDatabases.some((database) => database.type === "azure"),
       hasSnowflakePrivateKey: connection.type === "snowflake" && !!connection.snowflakeConnection?.privateKey,
       isDuckLake,
@@ -295964,7 +296080,17 @@ function buildEnvironmentMalloyConfig(connections = [], environmentPath = "", is
     return {
       lookupConnection: async (name) => {
         const metadata = getMetadataForLookup(assembled.metadata, name);
-        const connection = await resolveConnection(name, metadata);
+        let connection;
+        try {
+          connection = await resolveConnection(name, metadata);
+        } catch (error2) {
+          if (name !== undefined && !Object.hasOwn(assembled.pojo.connections, name)) {
+            throw new UnconfiguredConnectionError(name, {
+              cause: error2
+            });
+          }
+          throw error2;
+        }
         if (isDuckDBConnection(connection)) {
           await applyExtensionSessionSettings(connection);
         }
@@ -315837,12 +315963,18 @@ function describeParseFailure(message) {
   }
   return `does not parse (${message})`;
 }
+function unparsedTagMessage(parseError) {
+  return `${TAG_UNPARSED_PREFIX}${describeParseFailure(parseError)}, ${TAG_DISCARDED}`;
+}
+function isUnparsedDashboardTagFinding(message) {
+  return message.startsWith(TAG_UNPARSED_PREFIX) && message.endsWith(TAG_DISCARDED);
+}
 function lintUndiscoveredDashboard(facts) {
   const subject = dashboardSlug(facts.modelPath);
   const messages = [
     ...motlyParseErrors(facts.modelAnnotations),
     ...facts.queries.flatMap((query) => motlyParseErrors(query.annotations))
-  ].map((message) => `Tag ${describeParseFailure(message)}, so the whole tag is discarded ` + `and this file is treated as a shared include rather than a dashboard.`);
+  ].map(unparsedTagMessage);
   const findings = Array.from(new Set(messages), (message) => ({ subject, message, severity: "error" }));
   if (findings.length > 0)
     return findings;
@@ -315973,7 +316105,7 @@ async function compileTileGivens(tiles, materializer, fallbackRegistry, gatesOf,
   }
   return compiled;
 }
-var MAX_BUILDER_COLUMNS = 24, COMPONENT_FILE_SUFFIXES, MODEL_FILE_SUFFIX2 = ".malloy", PUBLISHER_OWNED_TAGS, RENDERER_CHILD_TAGS, UNKNOWN_RENDER_TAG, COMPOSITE_ARTIFACT_PROPERTIES, QUERY_ARTIFACT_PROPERTIES, ARTIFACT_PROPERTY_REPLACEMENTS, NAME_THE_TILES = `Name the views to show: tiles=["source -> view", …].`;
+var MAX_BUILDER_COLUMNS = 24, COMPONENT_FILE_SUFFIXES, MODEL_FILE_SUFFIX2 = ".malloy", PUBLISHER_OWNED_TAGS, RENDERER_CHILD_TAGS, UNKNOWN_RENDER_TAG, COMPOSITE_ARTIFACT_PROPERTIES, QUERY_ARTIFACT_PROPERTIES, ARTIFACT_PROPERTY_REPLACEMENTS, TAG_DISCARDED, TAG_UNPARSED_PREFIX = "Tag ", NAME_THE_TILES = `Name the views to show: tiles=["source -> view", …].`;
 var init_dashboard = __esm(() => {
   init_annotations();
   init_authorize();
@@ -315997,6 +316129,7 @@ var init_dashboard = __esm(() => {
     dashboard_columns: "A single query has no grid; a composite's width is `dashboard { columns=N }` beside its artifact tag.",
     dashboard: "The grid width is a sibling of the artifact tag, not a property of it: " + "close the artifact braces first, as `## artifact { … } dashboard { columns=N }`."
   };
+  TAG_DISCARDED = "so the whole tag is discarded and this file is treated as a shared " + "include rather than a dashboard.";
 });
 
 // ../../node_modules/lunr/lunr.js
@@ -320207,10 +320340,10 @@ var init_model_info = () => {};
 // src/service/filter.ts
 function parseFilterAnnotation(annotation) {
   const trimmed = annotation.trim();
-  if (!trimmed.startsWith(ANNOTATION_PREFIX)) {
+  if (!trimmed.startsWith(FILTER_ANNOTATION_PREFIX)) {
     return null;
   }
-  const body = trimmed.slice(ANNOTATION_PREFIX.length).trim();
+  const body = trimmed.slice(FILTER_ANNOTATION_PREFIX.length).trim();
   const tokens = tokenize2(body);
   let name;
   let dimension;
@@ -320379,7 +320512,7 @@ function tokenize2(input) {
   }
   return tokens;
 }
-var VALID_FILTER_TYPES, ANNOTATION_PREFIX = "#(filter)", ISO_DATE_RE, ISO_TIMESTAMP_RE, FilterValidationError;
+var VALID_FILTER_TYPES, FILTER_ANNOTATION_PREFIX = "#(filter)", ISO_DATE_RE, ISO_TIMESTAMP_RE, FilterValidationError;
 var init_filter = __esm(() => {
   VALID_FILTER_TYPES = new Set([
     "equal",
@@ -326155,6 +326288,18 @@ async function compileErrorOf(runnable) {
     return translatorMalloyError(error2);
   }
 }
+async function runStatementCount(runnable) {
+  let prepared;
+  try {
+    prepared = await runnable.getPreparedQuery();
+  } catch {
+    return;
+  }
+  return prepared?.model?.queries?.().unnamed;
+}
+function multipleRunStatementsError(count4) {
+  return new InvalidArgumentError(`The query has ${count4} run: statements; only one runs per call, so the ` + "others would be ignored. Send each as its own request. (source: and " + "query: definitions before a single run: are fine.)");
+}
 function bindingsAllowDegradeToLive(bindings) {
   return bindings.length > 0 && bindings.every((b3) => b3.origin === "preaggregate" || b3.freshnessFallback === "live");
 }
@@ -326393,13 +326538,34 @@ var init_model = __esm(() => {
       }
       this.modelInfo = modelInfo ?? (this.modelDef ? modelInfoOf(this.modelDef) : undefined);
       if (this.filterMap.size > 0) {
-        logger.warn(`Model "${packageName}/${modelPath}" uses deprecated #(filter) annotations. Migrate to given: — see https://github.com/malloydata/publisher/blob/main/docs/givens.md`, {
+        logger.warn(`Model "${packageName}/${modelPath}" uses deprecated #(filter) annotations. Migrate to given: — see ${GIVENS_DOCS_URL}`, {
           packageName,
           modelPath,
           filterSourceCount: this.filterMap.size
         });
       }
     }
+    declaredFilterSourceNames() {
+      if (!this.modelDef)
+        return [];
+      const ownFile = (url3) => {
+        try {
+          return decodeURIComponent(new URL(url3).pathname).endsWith(`/${this.modelPath}`);
+        } catch {
+          return false;
+        }
+      };
+      const names = [];
+      for (const entry of Object.values(this.modelDef.contents)) {
+        if (!isSourceDef9(entry))
+          continue;
+        const own2 = entry.annotations?.blockNotes ?? [];
+        const declares = own2.some((note) => note.text.trim().startsWith(FILTER_ANNOTATION_PREFIX) && note.at !== undefined && ownFile(note.at.url));
+        if (declares)
+          names.push(entry.as || entry.name);
+      }
+      return names.sort();
+    }
     getFilters(sourceName) {
       return this.filterMap.get(sourceName) ?? [];
     }
@@ -326679,6 +326845,14 @@ var init_model = __esm(() => {
         checkOnly: true
       });
     }
+    unboundGateDenial(runnable, givens, fallbackLabel, gateGivenNames) {
+      if (!this.queryHadRowLevelFilterAttached(runnable) || ![...gateGivenNames].some((name) => !(name in (givens ?? {})))) {
+        return;
+      }
+      const unbound = this.rowLevelFilteredRunnables.get(runnable)?.find((graft) => graft.callerJoinPath && graft.givenNames.some((name) => !(name in (givens ?? {}))));
+      recordRowLevelGateDecision("denied_by_gate", unbound ? "caller_join" : "entry_point");
+      return new AccessDeniedError(`Access denied for source "${unbound?.label ?? fallbackLabel}".`);
+    }
     queryHadRowLevelFilterAttached(runnable) {
       return this.rowLevelFilteredRunnables.has(runnable);
     }
@@ -327544,7 +327718,7 @@ ${query}`;
       const sourceInfos = data.sourceInfos;
       const givens = data.givens;
       const filterMap = data.filterMap ? new Map(data.filterMap) : undefined;
-      for (const warning of data.authorizeWarnings ?? []) {
+      for (const warning of options?.skipAuthorizeWarningLog ? [] : data.authorizeWarnings ?? []) {
         logger.warn(warning, { packageName, modelPath: data.modelPath });
       }
       if (!modelDef) {
@@ -327558,8 +327732,10 @@ ${query}`;
       model.compiledSourceText = data.modelSourceText;
       return model;
     }
-    static fromCompilationError(packageName, modelPath, modelType, error2) {
-      return new Model(packageName, modelPath, {}, modelType, undefined, undefined, undefined, undefined, undefined, undefined, error2);
+    static fromCompilationError(packageName, modelPath, modelType, error2, sourceText) {
+      const model = new Model(packageName, modelPath, {}, modelType, undefined, undefined, undefined, undefined, undefined, undefined, error2);
+      model.compiledSourceText = sourceText;
+      return model;
     }
     static deserializeCompilationError = deserializeError;
     getPath() {
@@ -328602,6 +328778,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
           throw queryCompileError(compileError, query);
         }
       }
+      const runCount = !sourceName && !queryName && query ? await runStatementCount(runnable) : undefined;
       const compiledSource = await this.resolveAuthorizeSourceFromRunnable(runnable);
       if (boundary === "deferred") {
         this.assertQueryBoundaryCompiled(compiledSource, query);
@@ -328615,6 +328792,16 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
         callerQueryText: query,
         callerJoins: callerRegion && liveRunnable ? { runnable: liveRunnable, region: callerRegion } : undefined
       });
+      if (runCount !== undefined && runCount > 1) {
+        this.queryExecutionHistogram.record(performance.now() - startTime, this.queryMetricAttributes({
+          environment: queryMetadataInput?.environment,
+          queryName,
+          sourceName,
+          status: "error",
+          servedFrom
+        }));
+        throw this.unboundGateDenial(runnable, givens, compiledSource ?? sourceName ?? "unknown", (this.rowLevelFilteredRunnables.get(runnable) ?? []).flatMap((graft) => graft.givenNames)) ?? multipleRunStatementsError(runCount);
+      }
       this.assertFilterGivens(givens);
       const maxRows = getMaxQueryRows();
       const maxBytes = getMaxResponseBytes();
@@ -328659,15 +328846,14 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
             status: "error",
             servedFrom
           }));
-          if (isGivenBindingFailure(err) && this.queryHadRowLevelFilterAttached(runnable) && [...this.authorizeReferencedGivenNames].some((name) => !(name in (givens ?? {})))) {
+          const gateDenial = isGivenBindingFailure(err) ? this.unboundGateDenial(runnable, givens, compiledSource ?? sourceName ?? "unknown", this.authorizeReferencedGivenNames) : undefined;
+          if (gateDenial) {
             logger.debug("Gate given unbound; denying opaquely", {
               environmentName: this.packageName,
               modelPath: this.modelPath,
               error: err instanceof Error ? err.message : String(err)
             });
-            const unbound = this.rowLevelFilteredRunnables.get(runnable)?.find((graft) => graft.callerJoinPath && graft.givenNames.some((name) => !(name in (givens ?? {}))));
-            recordRowLevelGateDecision("denied_by_gate", unbound ? "caller_join" : "entry_point");
-            throw new AccessDeniedError(`Access denied for source "${unbound?.label ?? compiledSource ?? sourceName ?? "unknown"}".`);
+            throw gateDenial;
           }
           const givenCode = err?.code;
           if (typeof givenCode === "string" && givenCode.startsWith("runtime-given-")) {
@@ -328685,6 +328871,9 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
             throw err;
           }
           const errorMessage = err instanceof Error ? err.message : String(err);
+          const accessFailure = databaseAccessFailure(err);
+          if (accessFailure)
+            throw accessFailure;
           logger.error("Query execution error", {
             error: err,
             errorMessage,
@@ -329144,6 +329333,9 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
             throw error2;
           }
           const errorMessage = error2 instanceof Error ? error2.message : String(error2);
+          const accessFailure = databaseAccessFailure(error2);
+          if (accessFailure)
+            throw accessFailure;
           if (errorMessage.trim() === "Model has no queries.") {
             return {
               type: "code",
@@ -358997,7 +359189,7 @@ async function getSchemasForBigQuery(connection) {
     }));
   } catch (error2) {
     logger.error(`Error getting schemas for BigQuery connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for BigQuery connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for BigQuery connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForPostgres(connection, malloyConnection) {
@@ -359018,7 +359210,7 @@ async function getSchemasForPostgres(connection, malloyConnection) {
     });
   } catch (error2) {
     logger.error(`Error getting schemas for Postgres connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for Postgres connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for Postgres connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForMySQL(connection) {
@@ -359096,7 +359288,7 @@ async function getSchemasForSnowflake(connection, malloyConnection) {
     if (error2 instanceof BadRequestError)
       throw error2;
     logger.error(`Error getting schemas for Snowflake connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for Snowflake connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for Snowflake connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForTrino(connection, malloyConnection) {
@@ -359151,7 +359343,7 @@ async function getSchemasForTrino(connection, malloyConnection) {
     if (error2 instanceof BadRequestError)
       throw error2;
     logger.error(`Error getting schemas for Trino connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for Trino connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for Trino connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForDatabricks(connection, malloyConnection) {
@@ -359207,7 +359399,7 @@ async function getSchemasForDatabricks(connection, malloyConnection) {
     if (error2 instanceof BadRequestError)
       throw error2;
     logger.error(`Error getting schemas for Databricks connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for Databricks connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for Databricks connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForDuckDB(connection, malloyConnection) {
@@ -359262,7 +359454,7 @@ async function getSchemasForDuckDB(connection, malloyConnection) {
     return schemas3;
   } catch (error2) {
     logger.error(`Error getting schemas for DuckDB connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for DuckDB connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for DuckDB connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForMotherDuck(connection, malloyConnection) {
@@ -359285,7 +359477,7 @@ async function getSchemasForMotherDuck(connection, malloyConnection) {
     });
   } catch (error2) {
     logger.error(`Error getting schemas for MotherDuck connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for MotherDuck connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for MotherDuck connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function getSchemasForDuckLake(connection, malloyConnection) {
@@ -359305,7 +359497,7 @@ async function getSchemasForDuckLake(connection, malloyConnection) {
     });
   } catch (error2) {
     logger.error(`Error getting schemas for DuckLake connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get schemas for DuckLake connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get schemas for DuckLake connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 var PUBLISHER_INTROSPECTION_TIMEOUT_MS = 60000;
@@ -359337,7 +359529,7 @@ async function fetchFromPublisherDataplane(connection, pathSuffix) {
     });
   } catch (error2) {
     const reason = error2?.name === "TimeoutError" ? `timed out after ${PUBLISHER_INTROSPECTION_TIMEOUT_MS}ms` : error2.message;
-    throw new Error(`Publisher dataplane request to ${redactUrlCredentials(url3)} failed ` + `for connection "${connection.name}": ${reason}`);
+    throw new Error(`Publisher dataplane request to ${redactUrlCredentials(url3)} failed ` + `for connection "${connection.name}": ${reason}`, { cause: error2 });
   }
   if (!response.ok) {
     const body = await response.text().catch(() => "");
@@ -359609,7 +359801,7 @@ async function listTablesForBigQuery(connection, schemaName, malloyConnection, t
     return results;
   } catch (error2) {
     logger.error(`Error getting tables for BigQuery schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for BigQuery schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for BigQuery schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForMySQL(connection, schemaName, malloyConnection, tableNames) {
@@ -359622,7 +359814,7 @@ async function listTablesForMySQL(connection, schemaName, malloyConnection, tabl
     return groupColumnRowsIntoTables(rows, (t2) => `${schemaName}.${t2}`);
   } catch (error2) {
     logger.error(`Error getting tables for MySQL schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for MySQL schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for MySQL schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForPostgres(connection, schemaName, malloyConnection, tableNames) {
@@ -359635,7 +359827,7 @@ async function listTablesForPostgres(connection, schemaName, malloyConnection, t
     return groupColumnRowsIntoTables(rows, (t2) => `${schemaName}.${t2}`);
   } catch (error2) {
     logger.error(`Error getting tables for Postgres schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for Postgres schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for Postgres schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForSnowflake(connection, schemaName, malloyConnection, tableNames) {
@@ -359665,7 +359857,7 @@ async function listTablesForSnowflake(connection, schemaName, malloyConnection,
     logger.error(`Error getting tables for Snowflake schema ${schemaName} in connection ${connection.name}`, { error: error2 });
     if (error2 instanceof BadRequestError)
       throw error2;
-    throw new Error(`Failed to get tables for Snowflake schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for Snowflake schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForTrino(connection, schemaName, malloyConnection, tableNames) {
@@ -359698,7 +359890,7 @@ async function listTablesForTrino(connection, schemaName, malloyConnection, tabl
     logger.error(`Error getting tables for Trino schema ${schemaName} in connection ${connection.name}`, { error: error2 });
     if (error2 instanceof BadRequestError)
       throw error2;
-    throw new Error(`Failed to get tables for Trino schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for Trino schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForDatabricks(connection, schemaName, malloyConnection, tableNames) {
@@ -359731,7 +359923,7 @@ async function listTablesForDatabricks(connection, schemaName, malloyConnection,
     logger.error(`Error getting tables for Databricks schema ${schemaName} in connection ${connection.name}`, { error: error2 });
     if (error2 instanceof BadRequestError)
       throw error2;
-    throw new Error(`Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForDuckDB(connection, schemaName, malloyConnection, tableNames) {
@@ -359776,7 +359968,7 @@ async function listTablesForDuckDB(connection, schemaName, malloyConnection, tab
     return groupColumnRowsIntoTables(rows, (t2) => `${schemaName}.${t2}`);
   } catch (error2) {
     logger.error(`Error getting tables for DuckDB schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for DuckDB schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for DuckDB schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForMotherDuck(connection, schemaName, malloyConnection, tableNames) {
@@ -359789,7 +359981,7 @@ async function listTablesForMotherDuck(connection, schemaName, malloyConnection,
     return groupColumnRowsIntoTables(rows, (t2) => `${schemaName}.${t2}`);
   } catch (error2) {
     logger.error(`Error getting tables for MotherDuck schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for MotherDuck schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for MotherDuck schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 async function listTablesForDuckLake(connection, schemaName, malloyConnection, tableNames) {
@@ -359804,7 +359996,7 @@ async function listTablesForDuckLake(connection, schemaName, malloyConnection, t
     return groupColumnRowsIntoTables(rows, (t2) => `${schemaName}.${t2}`);
   } catch (error2) {
     logger.error(`Error getting tables for DuckLake schema ${schemaName} in connection ${connection.name}`, { error: error2 });
-    throw new Error(`Failed to get tables for DuckLake schema ${schemaName} in connection ${connection.name}: ${error2.message}`);
+    throw new Error(`Failed to get tables for DuckLake schema ${schemaName} in connection ${connection.name}: ${error2.message}`, { cause: error2 });
   }
 }
 function extractErrorDataFromError(error2) {
@@ -359949,7 +360141,19 @@ function classifyDriverFailure(error2) {
     return error2;
   }
   const message = error2 instanceof Error ? error2.message : typeof error2 === "string" ? error2 : JSON.stringify(error2);
-  return driverErrorToPublisherError(message);
+  return databaseAccessFailure(error2) ?? driverErrorToPublisherError(message);
+}
+function sqlRunFailure(error2) {
+  if (error2 instanceof ConnectionError)
+    return error2;
+  return databaseAccessFailure(error2) ?? new ConnectionError(error2.message);
+}
+async function withDatabaseAccessClassified(list) {
+  try {
+    return await list();
+  } catch (error2) {
+    throw databaseAccessFailure(error2) ?? error2;
+  }
 }
 
 class ConnectionController {
@@ -359986,6 +360190,16 @@ class ConnectionController {
     }
   }
   async getMalloyConnection(environmentName, connectionName, packageName) {
+    try {
+      return await this.lookupMalloyConnection(environmentName, connectionName, packageName);
+    } catch (error2) {
+      if (error2 instanceof UnconfiguredConnectionError) {
+        throw new ConnectionNotFoundError(`Connection ${connectionName} not found`);
+      }
+      throw databaseAccessFailure(error2) ?? error2;
+    }
+  }
+  async lookupMalloyConnection(environmentName, connectionName, packageName) {
     const environment = await this.environmentStore.getEnvironment(environmentName, false);
     if (connectionName === "duckdb") {
       const packages = await environment.listPackages();
@@ -360022,7 +360236,9 @@ class ConnectionController {
         throw new TableNotFoundError(`Table ${tablePath} not found`);
       }
       if (typeof source === "string") {
-        throw driverErrorToPublisherError(source);
+        const prefix = `Error fetching schema for ${tablePath}: `;
+        const accessFailure = source.startsWith(prefix) ? databaseAccessFailure(new Error(source.slice(prefix.length))) : undefined;
+        throw accessFailure ?? driverErrorToPublisherError(source);
       }
       return {
         source: JSON.stringify(source),
@@ -360034,7 +360250,7 @@ class ConnectionController {
       };
     } catch (error2) {
       const classified = classifyDriverFailure(error2);
-      if (!(classified instanceof ConnectionError)) {
+      if (classified instanceof TableNotFoundError || classified instanceof InvalidArgumentError) {
         logger.warn("table not resolvable", {
           tableKey,
           tablePath,
@@ -360042,6 +360258,8 @@ class ConnectionController {
         });
         throw classified;
       }
+      if (databaseAccessFailure(classified))
+        throw classified;
       logger.error("fetchTableSchema error", {
         error: error2,
         tableKey,
@@ -360065,13 +360283,13 @@ class ConnectionController {
     const environment = await this.environmentStore.getEnvironment(environmentName, false);
     const connection = this.getApiConnectionForLookup(environment, connectionName);
     const malloyConnection = await this.getMalloyConnection(environmentName, connectionName, packageName);
-    return getSchemasForConnection(connection, malloyConnection);
+    return withDatabaseAccessClassified(() => getSchemasForConnection(connection, malloyConnection));
   }
   async listTables(environmentName, connectionName, schemaName, tableNames, packageName) {
     const environment = await this.environmentStore.getEnvironment(environmentName, false);
     const connection = this.getApiConnectionForLookup(environment, connectionName);
     const malloyConnection = await this.getMalloyConnection(environmentName, connectionName, packageName);
-    return listTablesForSchema(connection, schemaName, malloyConnection, tableNames);
+    return withDatabaseAccessClassified(() => listTablesForSchema(connection, schemaName, malloyConnection, tableNames));
   }
   async getConnectionSqlSource(environmentName, connectionName, sqlStatement, packageName) {
     const malloyConnection = await this.getMalloyConnection(environmentName, connectionName, packageName);
@@ -360191,7 +360409,7 @@ class ConnectionController {
         } catch (error2) {
           if (error2 instanceof PayloadTooLargeError || error2 instanceof ConnectionError)
             throw error2;
-          throw new ConnectionError(error2.message);
+          throw sqlRunFailure(error2);
         }
       }, getQueryTimeoutMs());
       return { data: JSON.stringify(streamed), queryCorrelationId };
@@ -360204,9 +360422,7 @@ class ConnectionController {
       try {
         return await malloyConnection.runSQL(sqlStatement, optionsWithSignal);
       } catch (error2) {
-        if (error2 instanceof ConnectionError)
-          throw error2;
-        throw new ConnectionError(error2.message);
+        throw sqlRunFailure(error2);
       }
     }, getQueryTimeoutMs());
     if (maxRows > 0 && result.rows.length > maxRows) {
@@ -360238,9 +360454,7 @@ class ConnectionController {
       } catch (error2) {
         if (signal.aborted)
           throw error2;
-        if (error2 instanceof ConnectionError)
-          throw error2;
-        throw new ConnectionError(error2.message);
+        throw sqlRunFailure(error2);
       }
     }, getQueryTimeoutMs());
   }
@@ -365375,6 +365589,15 @@ function getNotFoundError(resourceUriOrContext) {
     suggestions
   };
 }
+function getEnvironmentNotFoundError(lookup) {
+  const available = lookup.availableEnvironments.length > 0 ? `Available environments: ${lookup.availableEnvironments.join(", ")}.` : "This server has no environments loaded.";
+  return {
+    message: `Environment '${lookup.environmentName}' not found. ${available} Use a name from list_packages.`,
+    suggestions: [
+      "Environment names must match exactly, including case. Retrying with the same name fails the same way."
+    ]
+  };
+}
 function getInternalError(operation, error2) {
   const baseMessage = `An unexpected internal error occurred during ${operation}.`;
   const suggestions = [
@@ -365503,9 +365726,41 @@ function classifyToolError(operation, identifier, error2) {
       ]
     };
   }
+  if (error2 instanceof EnvironmentNotFoundError && error2.lookup) {
+    return getEnvironmentNotFoundError(error2.lookup);
+  }
   if (error2 instanceof EnvironmentNotFoundError || error2 instanceof PackageNotFoundError || error2 instanceof ModelNotFoundError || error2 instanceof NotQueryableError || error2 instanceof ConnectionNotFoundError) {
     return getNotFoundError(identifier);
   }
+  if (error2 instanceof ConnectionFailedError) {
+    logInternalFailure(`Database unreachable during ${operation}`, error2, "warn");
+    return {
+      message: `The database connection for ${identifier} is down: the database could not be reached, so the query never ran.`,
+      suggestions: [
+        "The query and the model are fine. Do not rewrite them.",
+        "Retry once. If it fails again, report that the database connection is down rather than changing the query."
+      ]
+    };
+  }
+  if (error2 instanceof UnconfiguredConnectionError) {
+    return {
+      message: `${error2.message}. The model for ${identifier} uses a connection this environment does not have, so the query never ran.`,
+      suggestions: [
+        "The query is fine. Do not rewrite it, and do not retry: it fails the same way until the connection exists.",
+        "Report that the environment is missing this connection; it was likely deleted or renamed."
+      ]
+    };
+  }
+  if (error2 instanceof ConnectionAuthError) {
+    logInternalFailure(`Connection credentials rejected during ${operation}`, error2, "warn");
+    return {
+      message: `The database rejected the connection's credentials for ${identifier}. The query never ran.`,
+      suggestions: [
+        "The query and the model are fine. Do not rewrite them, and do not retry: it fails the same way until the credentials are fixed.",
+        "Report that the connection's user, password, key or token needs updating."
+      ]
+    };
+  }
   if (error2 instanceof ServiceUnavailableError) {
     return {
       message: error2.message,
@@ -365568,7 +365823,7 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
   } catch (error2) {
     let errorDetails;
     if (error2 instanceof EnvironmentNotFoundError) {
-      errorDetails = getNotFoundError(`environment '${environmentName}'`);
+      errorDetails = classifyToolError("executeQuery (load model)", `environment '${environmentName}'`, error2);
     } else if (error2 instanceof PackageNotFoundError) {
       errorDetails = getNotFoundError(`package '${packageName}' in environment '${environmentName}'`);
     } else if (error2 instanceof ModelNotFoundError) {
@@ -377342,6 +377597,7 @@ init_notebook();
 init_dashboard();
 
 // src/service/notebook_lint.ts
+init_dashboard();
 init_notebook();
 init_motly();
 var KNOWN_KINDS = ["notebook", "dashboard", "text"];
@@ -377800,6 +378056,9 @@ function lintNotebookText(modelPath, text) {
     }
   }
 }
+function reportedByDashboardLint(finding, modelPath, dashboardFindings) {
+  return finding.code === "notebook-artifact-unparsed" && dashboardFindings.some((f5) => f5.model === modelPath && isUnparsedDashboardTagFinding(f5.message ?? ""));
+}
 function notebookLintProblems(modelPath, text, url3) {
   return lintNotebookText(modelPath, text).map((finding) => {
     const line = finding.line - 1;
@@ -379051,6 +379310,135 @@ class Package {
       throw error2;
     }
   }
+  static packageConfigFromOutcome(environmentName, packageName, outcome) {
+    return {
+      name: outcome.packageMetadata.name,
+      description: outcome.packageMetadata.description,
+      location: outcome.packageMetadata.location,
+      resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
+      explores: outcome.packageMetadata.explores,
+      queryableSources: outcome.packageMetadata.queryableSources,
+      manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
+      materialization: outcome.packageMetadata.materialization ?? {
+        schedule: null,
+        freshness: null
+      },
+      queryMetadata: outcome.packageMetadata.materialization?.queryMetadata ?? null,
+      scope: outcome.packageMetadata.scope ?? "package"
+    };
+  }
+  static async hydrateWorkerModels(outcome, ctx, onCompileError, afterHydrate) {
+    const { packageName, packagePath, malloyConfig, buildManifest } = ctx;
+    const models = new Map;
+    const renderTagWarnings = [];
+    const placeholder = (sm, err) => Model.fromCompilationError(packageName, sm.modelPath, sm.modelType, err, sm.modelPath === ctx.replacement?.modelPath ? ctx.replacement.source : undefined);
+    for (const sm of outcome.models) {
+      if (sm.compilationError) {
+        const err = Model.deserializeCompilationError(sm.compilationError);
+        if (onCompileError === "throw") {
+          logger.error("Model compilation failed", {
+            packageName,
+            modelPath: sm.modelPath,
+            error: err.message
+          });
+          throw err;
+        }
+        if (!ctx.compile) {
+          logger.warn("Model compilation failed during reload", {
+            packageName,
+            modelPath: sm.modelPath,
+            error: err.message
+          });
+        }
+        models.set(sm.modelPath, placeholder(sm, err));
+        continue;
+      }
+      let model;
+      try {
+        model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm, {
+          buildManifest,
+          skipAuthorizeWarningLog: ctx.compile
+        });
+      } catch (hydrateErr) {
+        if (!ctx.compile)
+          throw hydrateErr;
+        logger.warn("Model hydration failed during compile", {
+          packageName,
+          modelPath: sm.modelPath,
+          error: errMessage(hydrateErr)
+        });
+        renderTagWarnings.push({
+          model: sm.modelPath,
+          message: `The load-time checks could not read this model ` + `(${errMessage(hydrateErr)}), so its render-tag and dashboard findings ` + `are unknown rather than clean.`,
+          severity: "error"
+        });
+        models.set(sm.modelPath, placeholder(sm, hydrateErr instanceof Error ? hydrateErr : new Error(String(hydrateErr))));
+        continue;
+      }
+      let warnings;
+      try {
+        warnings = await model.validateRenderTags();
+      } catch (renderErr) {
+        if (onCompileError === "throw")
+          throw renderErr;
+        const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
+        logger.warn(`Render-tag validation failed during ${ctx.compile ? "compile" : "reload"}`, {
+          packageName,
+          modelPath: sm.modelPath,
+          error: err.message
+        });
+        if (ctx.compile) {
+          renderTagWarnings.push({
+            model: sm.modelPath,
+            message: `The render-tag check did not finish for this model, so ` + `its render-tag findings are unknown rather than clean.`,
+            severity: "warn"
+          });
+        }
+        models.set(sm.modelPath, placeholder(sm, err));
+        continue;
+      }
+      for (const w2 of warnings) {
+        renderTagWarnings.push({
+          model: sm.modelPath,
+          subject: w2.subject,
+          message: w2.message,
+          severity: w2.severity
+        });
+      }
+      await afterHydrate?.(sm);
+      models.set(sm.modelPath, model);
+    }
+    return { models, renderTagWarnings };
+  }
+  static async lintWorkerOutcome(environmentName, packageName, packagePath, malloyConfig, outcome, buildManifest, replacement) {
+    const { models, renderTagWarnings } = await Package.hydrateWorkerModels(outcome, {
+      packageName,
+      packagePath,
+      malloyConfig,
+      buildManifest,
+      compile: true,
+      replacement
+    }, "placeholder");
+    const pkg = new Package(environmentName, packageName, packagePath, Package.packageConfigFromOutcome(environmentName, packageName, outcome), [], models, malloyConfig);
+    try {
+      await pkg.discoverDashboards({ dryRun: true });
+    } catch (err) {
+      logger.warn("Dashboard lint failed during compile", {
+        packageName,
+        error: errMessage(err)
+      });
+      return {
+        renderTagWarnings,
+        dashboardWarnings: [
+          {
+            message: "The dashboard checks did not run, so their findings " + "are unknown rather than clean. Reload the package to " + 'see them. The cause is in the server log under "Dashboard ' + 'lint failed during compile".',
+            severity: "warn"
+          }
+        ]
+      };
+    }
+    return { renderTagWarnings, dashboardWarnings: pkg.dashboardWarnings };
+  }
   static async loadViaWorker(environmentName, packageName, packagePath, malloyConfig, startTime, manifestValidationTime) {
     const pool = getPackageLoadPool();
     const dispatchTime = performance.now();
@@ -379083,49 +379471,16 @@ class Package {
       modelCount: outcome.models.length,
       databaseCount: databases.length
     });
-    const packageConfig = {
-      name: outcome.packageMetadata.name,
-      description: outcome.packageMetadata.description,
-      location: outcome.packageMetadata.location,
-      resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}`,
-      explores: outcome.packageMetadata.explores,
-      queryableSources: outcome.packageMetadata.queryableSources,
-      manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
-      materialization: outcome.packageMetadata.materialization ?? {
-        schedule: null,
-        freshness: null
-      },
-      queryMetadata: outcome.packageMetadata.materialization?.queryMetadata ?? null,
-      scope: outcome.packageMetadata.scope ?? "package"
-    };
-    const models = new Map;
-    const renderTagWarnings = [];
+    const packageConfig = Package.packageConfigFromOutcome(environmentName, packageName, outcome);
+    let models;
+    let renderTagWarnings;
     try {
-      for (const sm of outcome.models) {
-        if (sm.compilationError) {
-          const err = Model.deserializeCompilationError(sm.compilationError);
-          logger.error("Model compilation failed", {
-            packageName,
-            modelPath: sm.modelPath,
-            error: err.message
-          });
-          throw err;
-        }
-        const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
-        for (const w2 of await model.validateRenderTags()) {
-          renderTagWarnings.push({
-            model: sm.modelPath,
-            subject: w2.subject,
-            message: w2.message,
-            severity: w2.severity
-          });
-        }
+      ({ models, renderTagWarnings } = await Package.hydrateWorkerModels(outcome, { packageName, packagePath, malloyConfig }, "throw", async (sm) => {
         if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
           const modelSource = await fs11.readFile(path13.join(packagePath, sm.modelPath), "utf-8");
           assertPersistNamesQuoted(modelSource, sm.modelPath);
         }
-        models.set(sm.modelPath, model);
-      }
+      }));
     } catch (err) {
       recordPackageLoadPhases(outcome.timings, packageLoadFailureStatus(err));
       throw err;
@@ -379303,6 +379658,7 @@ class Package {
       ...this.droppedPersistWarnings(),
       ...this.persistRefusalWarnings(),
       ...this.emptyDiscoveryWarnings(),
+      ...this.legacyFormatWarnings(),
       ...this.brokenSurfaceWarnings(),
       ...this.surfaceWidenedWarning !== undefined ? [{ message: this.surfaceWidenedWarning }] : [],
       ...this.persistenceCollisionWarnings().map((message) => ({ message })),
@@ -379646,6 +380002,28 @@ class Package {
     }
     return warnings;
   }
+  legacyFormatWarnings() {
+    const notebooks = [];
+    const filters = [];
+    for (const [modelPath, model] of this.models) {
+      if (modelPath.endsWith(NOTEBOOK_FILE_SUFFIX)) {
+        notebooks.push({
+          model: modelPath,
+          severity: "warn",
+          message: `${modelPath} is a .malloynb notebook, a deprecated format. ` + `Convert it to a .malloy notebook under notebooks/.`
+        });
+      }
+      for (const name of model.declaredFilterSourceNames()) {
+        filters.push({
+          model: modelPath,
+          subject: name,
+          severity: "warn",
+          message: `Source ${name} uses deprecated #(filter) annotations. Replace them ` + `with given: runtime parameters (see ${GIVENS_DOCS_URL}).`
+        });
+      }
+    }
+    return [...notebooks, ...filters];
+  }
   brokenSurfaceWarnings() {
     const exploreSet = this.exploreSet();
     if (!exploreSet || exploreSet.size === 0)
@@ -379751,40 +380129,12 @@ class Package {
       throw new ServiceUnavailableError(`Package-load worker pool unavailable: ${realError.message}`, { cause: realError });
     }
     assertRequiredStagesAvailable(outcome.packageMetadata.retrieval ?? DEFAULT_PACKAGE_RETRIEVAL, llmConfigured());
-    const nextModels = new Map;
-    const renderTagWarnings = [];
-    for (const sm of outcome.models) {
-      if (sm.compilationError) {
-        const err = Model.deserializeCompilationError(sm.compilationError);
-        logger.warn("Model compilation failed during reload", {
-          packageName: this.packageName,
-          modelPath: sm.modelPath,
-          error: err.message
-        });
-        nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
-      } else {
-        const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm, { buildManifest });
-        try {
-          for (const w2 of await model.validateRenderTags()) {
-            renderTagWarnings.push({
-              model: sm.modelPath,
-              subject: w2.subject,
-              message: w2.message,
-              severity: w2.severity
-            });
-          }
-          nextModels.set(sm.modelPath, model);
-        } catch (renderErr) {
-          const err = renderErr instanceof Error ? renderErr : new Error(String(renderErr));
-          logger.warn("Render-tag validation failed during reload", {
-            packageName: this.packageName,
-            modelPath: sm.modelPath,
-            error: err.message
-          });
-          nextModels.set(sm.modelPath, Model.fromCompilationError(this.packageName, sm.modelPath, sm.modelType, err));
-        }
-      }
-    }
+    const { models: nextModels, renderTagWarnings } = await Package.hydrateWorkerModels(outcome, {
+      packageName: this.packageName,
+      packagePath: this.packagePath,
+      malloyConfig: this.malloyConfig,
+      buildManifest
+    }, "placeholder");
     this.models = nextModels;
     this.pushStorageServeBindingsToModels();
     this.pushServeDestinationConfigToModels();
@@ -379854,9 +380204,9 @@ class Package {
       return false;
     return this.exploresDeclared();
   }
-  async claimsToBeADashboard(modelPath) {
+  async claimsToBeADashboard(model, modelPath) {
     try {
-      const source = await fs11.readFile(safeJoinUnderRoot(this.packagePath, modelPath), "utf8");
+      const source = model.getCompiledSourceText() ?? await fs11.readFile(safeJoinUnderRoot(this.packagePath, modelPath), "utf8");
       return hasArtifactLineOutsideBlocks(source, ANY_ARTIFACT_NOTE);
     } catch {
       return false;
@@ -379870,13 +380220,14 @@ class Package {
       }
       if (!model.getCompilationError())
         return untagged;
-      const source = await fs11.readFile(safeJoinUnderRoot(this.packagePath, modelPath), "utf8");
+      const source = model.getCompiledSourceText() ?? await fs11.readFile(safeJoinUnderRoot(this.packagePath, modelPath), "utf8");
       return claimsToBeANotebook(source) ? documentKind(modelPath, artifactKindInText(source)) : untagged;
     } catch {
       return untagged;
     }
   }
-  async discoverDashboards() {
+  async discoverDashboards(options = {}) {
+    const during = options.dryRun ? { during: "compile" } : {};
     const discovered = new Map;
     const factsByPath = new Map;
     const allFacts = new Map;
@@ -379898,6 +380249,7 @@ class Package {
       } catch (err) {
         logger.warn("Reading a model's dashboard facts failed", {
           packageName: this.packageName,
+          ...during,
           modelPath,
           error: errMessage(err)
         });
@@ -379918,6 +380270,7 @@ class Package {
         } catch (err) {
           logger.warn("Notebook layout discovery failed", {
             packageName: this.packageName,
+            ...during,
             modelPath,
             error: errMessage(err)
           });
@@ -379932,6 +380285,7 @@ class Package {
         } catch (err) {
           logger.warn("Dashboard discovery failed", {
             packageName: this.packageName,
+            ...during,
             modelPath,
             error: errMessage(err)
           });
@@ -379947,10 +380301,11 @@ class Package {
         }
       } else {
         const error2 = model.getCompilationError();
-        if (!error2 || !await this.claimsToBeADashboard(modelPath)) {
-          if (!error2 && await this.claimsToBeADashboard(modelPath)) {
+        if (!error2 || !await this.claimsToBeADashboard(model, modelPath)) {
+          if (!error2 && await this.claimsToBeADashboard(model, modelPath)) {
             logger.warn("Dashboard file produced no facts and no error", {
               packageName: this.packageName,
+              ...during,
               modelPath
             });
             droppedByError.push({ modelPath, name });
@@ -379989,7 +380344,7 @@ class Package {
       if (!model)
         continue;
       try {
-        dashboardFileText.set(manifest.entryFile, await model.getFileText(this.packagePath));
+        dashboardFileText.set(manifest.entryFile, model.getCompiledSourceText() ?? await model.getFileText(this.packagePath));
       } catch {
         dashboardFileText.set(manifest.entryFile, "");
       }
@@ -380008,10 +380363,6 @@ class Package {
     }
     this.notebookFileText = notebookFileText;
     this.applyQueryBoundaryToModels();
-    this.notebookWarnings = [
-      ...this.attachNotebookCells(),
-      ...await this.lintNotebookFiles()
-    ];
     this.lintInputs = {
       factsByPath,
       allFacts,
@@ -380020,7 +380371,13 @@ class Package {
       droppedByError,
       shadowed
     };
-    await this.relintDashboards();
+    await this.relintDashboards({ log: !options.dryRun });
+    if (!options.dryRun) {
+      this.notebookWarnings = [
+        ...this.attachNotebookCells(),
+        ...await this.lintNotebookFiles()
+      ];
+    }
   }
   attachNotebookCells() {
     const warnings = [];
@@ -380065,7 +380422,7 @@ class Package {
         }
       }
       for (const finding of lintNotebookText(modelPath, text)) {
-        if (finding.code === "notebook-artifact-unparsed" && !this.isServedNotebook(modelPath))
+        if (reportedByDashboardLint(finding, modelPath, this.dashboardWarnings))
           continue;
         logger.warn("Notebook lint", {
           packageName: this.packageName,
@@ -380082,11 +380439,13 @@ class Package {
     return warnings;
   }
   lintInputs;
-  async relintDashboards() {
+  async relintDashboards(options = {}) {
     const inputs = this.lintInputs;
     if (!inputs)
       return;
     this.dashboardWarnings = await this.lintDashboards(inputs.factsByPath, inputs.allFacts, inputs.unconventionalSlugs, inputs.dashboardSlugs, inputs.droppedByError, inputs.shadowed);
+    if (options.log === false)
+      return;
     for (const warning of this.dashboardWarnings) {
       logger.warn("Dashboard lint", {
         packageName: this.packageName,
@@ -380740,6 +381099,7 @@ ${source}` : source ?? "";
             problems.push(tagged);
           }
         };
+        const { renderTagWarnings, dashboardWarnings } = await Package.lintWorkerOutcome(this.environmentName, packageName, packagePath, pkg.getMalloyConfig(), outcome, boundManifestEntries, source === undefined ? undefined : { modelPath: modelName, source });
         for (const compiled of outcome.models) {
           if (compiled.problems) {
             collect(compiled.problems, compiled.modelPath);
@@ -380764,9 +381124,20 @@ ${source}` : source ?? "";
             return;
           }));
           if (lintText !== undefined) {
-            collect(notebookLintProblems(compiled.modelPath, lintText, pathToFileURL3(path14.join(packagePath, compiled.modelPath)).toString()), compiled.modelPath);
+            collect(notebookLintProblems(compiled.modelPath, lintText, pathToFileURL3(path14.join(packagePath, compiled.modelPath)).toString()).filter((problem) => !reportedByDashboardLint(problem, compiled.modelPath, dashboardWarnings)), compiled.modelPath);
           }
         }
+        const asProblem = (warning, code) => ({
+          severity: warning.severity ?? "warn",
+          message: warning.subject ? `${warning.subject}: ${warning.message}` : warning.message,
+          code
+        });
+        for (const warning of renderTagWarnings) {
+          collect([asProblem(warning, "render-tag")], warning.model);
+        }
+        for (const warning of dashboardWarnings) {
+          collect([asProblem(warning, "dashboard-lint")], warning.model);
+        }
         if (source !== undefined && outcome.replacementMatchedExisting === false) {
           collect([
             {
@@ -381919,7 +382290,7 @@ ${query}`);
 // src/service/environment_store.ts
 init_google_credentials();
 // package.json
-var version2 = "0.9.6";
+var version2 = "0.9.7";
 
 // src/version.ts
 var SERVER_VERSION = version2;
@@ -382152,6 +382523,7 @@ class EnvironmentStore {
   initError = null;
   environmentMutexes = new Map;
   publisherConfigIsFrozen;
+  mcpIncludeHiddenFilesAndSources;
   finishedInitialization;
   isInitialized = false;
   storageManager;
@@ -382164,6 +382536,7 @@ class EnvironmentStore {
   inPlaceEnvs = new Set;
   constructor(serverRootPath) {
     this.serverRootPath = serverRootPath;
+    this.mcpIncludeHiddenFilesAndSources = isMcpIncludeHiddenFilesAndSources(serverRootPath);
     this.gcsClient = new Storage2;
     const watchEnvList = (process.env.PUBLISHER_WATCH || "").split(",").map((s2) => s2.trim()).filter((s2) => s2.length > 0);
     for (const envName of watchEnvList)
@@ -382746,7 +383119,10 @@ class EnvironmentStore {
       const environmentConfig = environmentManifest.environments.find((e2) => e2.name === environmentName);
       const environmentPath = existingEnvironment?.metadata.location || environmentConfig?.packages[0]?.location;
       if (!environmentPath) {
-        throw new EnvironmentNotFoundError(`Environment "${environmentName}" could not be resolved to a path.`);
+        throw new EnvironmentNotFoundError(`Environment "${environmentName}" could not be resolved to a path.`, {
+          environmentName,
+          availableEnvironments: Array.from(this.environments.keys()).sort()
+        });
       }
       return await this.addEnvironment({
         name: environmentName,
@@ -387539,7 +387915,7 @@ var compileShape = {
   packageName: exports_external.string().describe("Package containing the model, as list_packages returns it."),
   modelPath: exports_external.string().describe("Path to the .malloy model the check targets: the namespace the source is appended to (scope append), or the file it replaces (scope file / package-with-source)."),
   source: exports_external.string().optional().describe("The Malloy source to validate; never executed. Required at scope append/file; optional at scope package (a what-if replacement for modelPath)."),
-  scope: exports_external.enum(["append", "file", "package"]).optional().describe(`What source means. "append" (default): append to modelPath. "file": compile AS modelPath to validate an edit. "package": run reload's worker compiler over all .malloy/.malloynb files without serving the result; an optional source replaces modelPath.`),
+  scope: exports_external.enum(["append", "file", "package"]).optional().describe('What source means. "append" (default): append to modelPath. "file": compile AS modelPath to validate an edit. "package": dry-run a reload over all .malloy/.malloynb files without serving the result, including its render-tag and dashboard checks; an optional source replaces modelPath.'),
   includeSql: exports_external.boolean().optional().describe("When true and the source ends in a runnable query, also return the generated SQL for inspection. The query is still not run."),
   givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any authorize gate on the target model, whether or not includeSql is set.")
 };
@@ -387548,7 +387924,7 @@ var COMPILE_DESCRIPTION = `Compile-check Malloy without running a query. Use thi
 ## Scopes (the scope parameter)
 - "append" (default): append source to modelPath. Use for NEW definitions; existing definitions report "Cannot redefine". Positions refer to the concatenated file (a ## artifact document reads only your text, and no document is returned). Refused here, with a 400: import, connection.table(...), connection.sql(...), raw-SQL functions, given: declarations (reading the model's givens as $NAME is fine), ##! flags, and text that does not stand alone as top-level Malloy — a bare view:/dimension:/measure:, or anything parsing only as a continuation of the model's last statement. A ## artifact document also refuses # image, # link and @env.; plain text keeps them. Wrap a view body in a top-level query:, a field in a throwaway source: check is <source> extend { … }; use "file" or "package" for a model that declares data roots.
 - "file": compile source AS modelPath. Use to validate an EDIT before saving; positions match the submitted file.
-- "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and authorize gates outside prose still apply to caller text. A missing exact path is warned and treated as a new file. Save and call reload_package to serve a clean edit.
+- "package": dry-run a reload of every .malloy/.malloynb file without serving the result, with its render-tag and dashboard checks (no position) but not its other warnings. A source replaces modelPath so importers see the edit. Diagnostics may name hidden files; no rows or SQL are returned, and authorize gates still apply to caller text. An unmatched path is warned and treated as new. Save, then reload_package to serve it.
 
 ## Parameters
 - environmentName, packageName, modelPath: required. source: required at append/file, optional at package. Caller source may not declare an access-control gate outside prose — neither #(authorize) nor #(access_filter).
@@ -387715,6 +388091,22 @@ init_dashboard();
 // src/mcp/query_envelope.ts
 init_json_utils();
 var MAX_RESULT_CHARS = 90000;
+function dedupeAnnotations(annotations) {
+  let seen = new Set;
+  const kept = [];
+  for (let i2 = annotations.length - 1;i2 >= 0; i2--) {
+    const annotation = annotations[i2];
+    if (seen.has(annotation.value))
+      continue;
+    kept.push(annotation);
+    if (annotation.value.includes(":=")) {
+      seen = new Set;
+      continue;
+    }
+    seen.add(annotation.value);
+  }
+  return kept.reverse();
+}
 function serialize(envelope) {
   return JSON.stringify(envelope, bigIntReplacer, 2);
 }
@@ -387728,13 +388120,13 @@ function buildQueryEnvelope(rows, rowLimit, result, renderLogErrors = [], limit
       annotations: result.annotations ?? [],
       connection_name: result.connection_name,
       ...result.model_annotations !== undefined && {
-        model_annotations: result.model_annotations
+        model_annotations: dedupeAnnotations(result.model_annotations)
       },
       ...result.query_timezone !== undefined && {
         query_timezone: result.query_timezone
       },
       ...result.source_annotations !== undefined && {
-        source_annotations: result.source_annotations
+        source_annotations: dedupeAnnotations(result.source_annotations)
       }
     },
     _query_row_limit: rowLimit,
@@ -387806,19 +388198,22 @@ var executeQueryShape = {
   environmentName: exports_external.string().describe("Environment name, as list_packages returns it."),
   packageName: exports_external.string().describe("Package containing the model, as list_packages returns it."),
   modelPath: exports_external.string().describe("Path to the .malloy model file"),
-  query: exports_external.string().optional().describe(`Ad-hoc Malloy query code. Runs in restricted mode: it may not use ${RESTRICTED_CONSTRUCTS} — put those in a model file and reload instead.`),
+  query: exports_external.string().optional().describe(`Ad-hoc Malloy query code with exactly one run: statement; source: and query: definitions may come before it. Text with more than one run: is refused, so send each as its own call. Runs in restricted mode: it may not use ${RESTRICTED_CONSTRUCTS} — put those in a model file and reload instead.`),
   sourceName: exports_external.string().optional().describe("Source name for a view. A NAME, not Malloy code: one name exactly as get_context returned it, sent bare (the server quotes it, so a hyphen or a reserved word is fine — do not add backticks yourself). Anything richer, such as a parameterized source or an inline extension, goes in query."),
   queryName: exports_external.string().optional().describe("Named query or view. A NAME, not Malloy code, on the same terms as sourceName: one view name as get_context returned it. A dotted path (carriers.by_name), a refinement (by_carrier + { limit: 10 }), or anything containing a newline goes in query instead."),
   filterParams: exports_external.record(exports_external.union([exports_external.string(), exports_external.array(exports_external.string())])).optional().describe("Filter parameter values keyed by filter name. Used with sources that declare #(filter) annotations."),
-  givens: exports_external.record(exports_external.unknown()).optional().describe("Per-query given values that override model defaults. Keys are given names declared in the model's given: block.")
+  givens: exports_external.record(exports_external.unknown()).optional().describe("Per-query given values that override model defaults. Keys are given names declared in the model's given: block."),
+  includeHiddenFilesAndSources: exports_external.boolean().optional().describe("Set true to also run the files and sources this package's index.malloy hides, so an author can test them without publishing them. It never bypasses #(authorize) or #(access_filter). Default false.")
 };
+var { includeHiddenFilesAndSources: _authoringOnly, ...curatedShape } = executeQueryShape;
 var EXECUTE_QUERY_DESCRIPTION = `Run a Malloy query against a model and return the rows. Takes either ad-hoc Malloy in query, or a named view/query via queryName (with sourceName for a view).
 
 ## Contract rules
 - Check _limit_hit before reporting any total, count, or "top N". True means the server's default cap cut the result off and more rows exist, so what came back is a partial set, not the answer.
 - Never sum or count the returned rows to state a total when _limit_hit or _rows_truncated is set. Aggregate in the query instead.
 - _returned_rows: 0 with _rows_truncated set means one row was too large to send, NOT that nothing matched. Do not report it as an empty result.
-- Use source, view, and field names exactly as get_context returned them. sourceName/queryName take one NAME each, never Malloy code — they are quoted for you, so send even a hyphenated name bare, and put anything richer (a dotted path, a refinement, a second statement) in query.
+- Use source, view, and field names exactly as get_context returned them. sourceName/queryName take one NAME each, never Malloy code — they are quoted for you, so send even a hyphenated name bare, and put anything richer (a dotted path, a refinement, a definition) in query.
+- One run: per call. query may define sources and queries first, but more than one run: statement is refused rather than run; send each question as its own call.
 - query is RESTRICTED: no raw SQL/import/##! (see its param doc).
 
 ## Response
@@ -387834,7 +388229,8 @@ A JSON object, the same shape Credible's execute_query and an in-package data ap
 
 Values above 2^53 are returned as JSON strings so their digits survive.`;
 function registerExecuteQueryTool(mcpServer, environmentStore) {
-  mcpServer.tool("execute_query", EXECUTE_QUERY_DESCRIPTION, executeQueryShape, async (params) => {
+  const offerHidden = environmentStore.mcpIncludeHiddenFilesAndSources === true;
+  mcpServer.tool("execute_query", EXECUTE_QUERY_DESCRIPTION, offerHidden ? executeQueryShape : curatedShape, async (params) => {
     const {
       environmentName,
       packageName,
@@ -387845,6 +388241,7 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
       filterParams,
       givens
     } = params;
+    const includeHiddenFilesAndSources = offerHidden && params.includeHiddenFilesAndSources === true;
     logger.info("[MCP Tool executeQuery] Received params:", { params });
     const hasAdhocQuery = !!query;
     const hasNamedQuery = !!queryName;
@@ -387887,7 +388284,7 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
         rowLimit,
         rowLimitSource,
         queryCorrelationId
-      } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact") : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact"), getQueryTimeoutMs());
+      } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact", false, includeHiddenFilesAndSources) : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact", false, includeHiddenFilesAndSources), getQueryTimeoutMs());
       const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist15(), 1));
       const renderLogs = filterPublisherOwnedRenderLogs(validateRenderTags2(result), modelPath);
       const resultUri = buildMalloyUri({
@@ -387934,7 +388331,7 @@ var reloadShape = {
 var RELOAD_FAILURE_IS_SAFE = "A reload that fails to compile leaves your files on disk alone and keeps serving the previously compiled model, returning the compile errors.";
 var RELOAD_DESCRIPTION = `Reload a package so edits to its model files on disk are picked up, making newly added or changed sources, views, and named queries resolvable by execute_query WITHOUT restarting the server. Publisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until the package is reloaded. Use this to close the edit -> run loop after saving a model change.
 
-${RELOAD_FAILURE_IS_SAFE} Running compile_model first is still the faster way to see diagnostics, and it keeps a broken model from ever reaching the reload.
+${RELOAD_FAILURE_IS_SAFE} Running compile_model at scope "package" first is the faster way to see diagnostics, and it keeps a broken model from ever reaching the reload. It reports the compile errors, render-tag findings and dashboard findings a reload would; every other package warning (storage, persist, materialization, explores, surface and discovery warnings, and deprecated-format warnings for .malloynb notebooks and #(filter) sources, among others) appears only here.
 
 ## Parameters
 - environmentName, packageName (required): the package to recompile. Use the names get_context returns.
@@ -392842,6 +393239,7 @@ Find the right entities before writing any query.
 
 - If you do not already know which package to work in, confirm the environment and package with the user before continuing.
 - **Load \`skill:malloy-phrase-detection\`**, then call \`get_context\` with entity targets that describe the fields the question needs: a \`measure\` for the metric, a \`dimension\` for each breakdown or filter, and a \`view\` if the question sounds like a canned report. **Every target carries a \`search_text\`**: a target without one enumerates a capped catalog instead of searching, and the cap is blind to your question. The tool description covers what comes back and how to narrow or browse.
+- **A value the user typed (a brand, a city, a product name) is not yet a filter.** Resolve it to the string the data stores before you filter on it: a \`dimensional_value\` target in that \`get_context\` call, or, where value search isn't available, a query of the dimension's distinct values. "ray ban" may be stored as \`Ray-Ban\` and \`Ray-Ban Junior\`; a \`where:\` on the user's own spelling, or a regex guessed from it, can match nothing and still return a result.
 - Read the documentation on each returned entity: on an entity it is \`description\`, on a source it is \`docs\`, often with a short \`one_line_summary\` and sometimes a longer \`summary\`. Which of these a server fills varies, so read the ones your response actually carries and do not go hunting for one that is absent. Authors write it as \`#(doc)\` in the model, but the response never uses that label, so do not go looking for that either. It is where grain, units, null handling, and any source-level filters are described. Confirm the exact field names against the results before using them.
 - **Read the source's own documentation too, not just each field's.** The source-level \`docs\` often defines the grain, the universe of rows it represents, how joins behave, and source-level filters or assumptions that apply to every query rooted on it. Factor both the source and the field docstrings into how you build and later verify the query.
 - **Do not rebuild a view that already exists from its description.** A description says what a view does, not how; rebuilding from prose loses what prose does not carry: a denominator, a \`partition_by\`, an exact filter. Set \`entity_name\` in the scope to get that entity on its own instead of a ranked sweep. That also returns its \`code\` (for a view, its definition as written): read it first, with the source's \`docs\` and the entity's \`description\`. Then RUN the view and adapt its output rather than writing a replacement; to change a band width or grain, change that one thing and keep the rest. Where a server returns no \`code\`, the docs plus the view's own output are what you have, and a calculation you cannot see is a reason to reuse the view rather than guess at it.
@@ -393010,8 +393408,12 @@ endpoint named beside it.
    already-defined errors that reads as broken Malloy rather than a wrong scope. **Editing a shared
    include wants \`"scope": "package"\`**, which recompiles every file as saved: \`file\` only checks the
    one you are editing, so renaming a source in \`_shared.malloy\` passes it while breaking every
-   dashboard that imports it. A clean compile is not a working dashboard: some tag mistakes surface
-   at step 6, and some only when you look at the page in step 7. **A not-yet-saved dashboard wants
+   dashboard that imports it. **\`"scope": "file"\` does not check tiles, drills or suggest queries.**
+   To see those before you save, compile the same text at \`"scope": "package"\` with \`modelPath\` set
+   to the dashboard's path: it runs the lint step 6 reads and returns its findings with code
+   \`dashboard-lint\` (or \`render-tag\`), each at the severity a load gives it, so an error finding
+   makes the compile an error too. Even that is not a working dashboard: some mistakes only show when
+   you look at the page in step 7. **A not-yet-saved dashboard wants
    \`"scope": "file"\` too**, not the default: a dashboard file opens with an \`import\`, and the
    default \`append\` scope refuses one, so the new-file case fails on the import and again on the
    path that does not exist yet. \`append\` is for a fragment checked against a model that is
@@ -393067,8 +393469,9 @@ nest's pipeline starts from its own query's source and there is no way to combin
 
 Three things the form costs, so you are not surprised by them:
 
-- **A tile expression is a string in an annotation, so the compiler never checks it.** Rename a view
-  and the file still compiles; the tile fails at package load. Read the lint (step 6).
+- **A tile expression is a string in an annotation, so the Malloy compiler never checks it.** Rename
+  a view and the file still compiles; the tile fails at package load. The lint names it: at step 6,
+  or before saving with \`compile_model\` at \`"scope": "package"\` (step 5).
 - **No per-parent-row grouping.** There is no parent query to repeat a grid over.
 - **Filtering lives on the tiles**, not on the page: each view's \`+ { where: ... }\` names the
   controls it answers to. Below is why, and the one thing that does not work.
@@ -393255,6 +393658,10 @@ value, that the prefix "is not a well-formed route", because a route ends at the
 complaint is a **compile** diagnostic on a compile that still succeeds, not a package warning, so
 step 6 will not show it. Pick by which reader you care about.
 
+\`dimension=\` can name a field of a directly joined source (one join level). Quote it, because
+an unquoted dotted value does not parse as a tag:
+\`suggest { source=sales dimension="products.department" }\`.
+
 On the \`query=\` form of \`suggest\`, \`dimension\` names a column of that query's output, not a path into
 the model, and only the last segment of it is read. Match it to the suggest query's \`group_by\`. When it
 matches no column the picker falls back to the first column of the result: silently right for a
@@ -394098,7 +394505,7 @@ This is the modeling-time consequence of ignoring \`skill:malloy-define\`'s scop
 
 ## Thresholds Are Decisions, Not Syntax
 
-Before writing a \`pick\` expression or filtered measure with a numeric cutoff, see \`skill:malloy-model\` § Key Rules: every boundary must be user-supplied, distribution-derived (query the percentiles first), or explicitly flagged as an assumption in its \`#(doc)\`. Never invent one silently.
+Before writing a \`pick\` expression or filtered measure with a numeric cutoff, see \`skill:malloy-model\` § Key Rules: every boundary must be user-supplied, distribution-derived (query the percentiles first: Malloy has no percentile function, so use the **Tier boundaries** query in \`skill:malloy-discover\`), or explicitly flagged as an assumption in its \`#(doc)\`. Never invent one silently.
 
 ## \`except:\` Removes Fields From Namespace Entirely
 
@@ -395736,7 +396143,7 @@ dimension: is_complete is status = 'complete'
 | \`type: min\` / \`max\` | \`min(field)\` / \`max(field)\` | Direct mapping |
 | \`type: number\` | Derived measure expression | Usually a ratio; use \`nullif()\` for division |
 | \`type: list\` | No direct equivalent | Flag for alternative approach |
-| \`type: percentile\` | No measure form: Malloy has no \`percentile\` function | Compute the value with the two-stage query in \`skill:malloy-discover\` § Example Queries, and tell the user it is a query result, not a reusable measure |
+| \`type: percentile\` / \`type: median\` | No measure form: Malloy has no \`percentile\` function | Compute the value with the two-stage query in \`skill:malloy-discover\` § Example Queries, and tell the user it is a query result, not a reusable measure |
 
 **Filtered measures:** \`filters:\` → Malloy \`{ where: }\` syntax:
 \`\`\`
@@ -395863,13 +396270,85 @@ Present coverage in this order:
     },
     {
       name: "malloy-materialization",
-      description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, tune what to persist, or asks why a persist source isn't building.",
-      body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n**Deciding what to persist, what to stop persisting, and how to schedule it** (making a package cheaper or faster): read `reference/tuning.md`. It reads the materialization history with the `malloy-pub` CLI and proposes changes; it recommends and does not edit.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n   - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n   - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"materialization\": {\n       \"scope\": \"package\",  // default; \"version\" = each published version owns its own tables\n       \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n       \"queryMetadata\": { \"team\": \"finance\" }  // tags the build's backend statements\n     }\n   }\n   ```\n   Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n   - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n   - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n   - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n   - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting a `#(access_filter)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, naming it. The build skips a refused\n  source, records it on the run (`metadata.refusedSources`), and builds the rest of the package. A run fails\n  on a refusal only when every authored source it targeted was refused, or when `sourceNames` named this one;\n  a refused rollup never fails it. (`#@ persist storage=<name>` is the tier that materializes into a separate\n  registered storage destination and serves from there, rather than building in the source's own connection;\n  `#@ preaggregate` stores a rollup Publisher derives from a measure you annotated with a grain, rather than a\n  source you wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards\n  even in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n  **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n  the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n  through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n  itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n  re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n  and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n  decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n  keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n  so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n  **layers**, which do not combine - not about siblings, below. Prefer the\n  **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n  the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n  it package-wide forces every other persisted source to recompute once stale too. And **a\n  content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n  content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n  two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n  policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n  cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n  `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n  layer rule above does not carry over here. If two sources need genuinely different windows, give them\n  genuinely different SQL.\n\n  Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n  the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n  manifest-assembly time typically applies it when a version's manifest is next published rather than\n  retroactively to manifests already distributed - so you can declare the window correctly and not have it\n  in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n  decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n  window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n  `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n  re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n  rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **A `#(access_filter)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting a `#(access_filter)`-gated source**.\n\n## Reference files over MCP\n\nThis skill's `reference/` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read `reference/<name>.md`, get the prompt named `malloy-materialization/<name>` instead.\n\nAvailable: tuning."
+      description: "Add, confirm and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table, in the source's own warehouse or in a storage destination. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, build one stored rollup over another, tune what to persist, or asks why a persist source isn't building or isn't being read.",
+      body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of four things - no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**), a build ran and failed or lost that one source (a run can finish ready with `failures`; read them), the model declaring the source has no `##! experimental.persistence` flag and nothing flagged imports it (it is skipped without a warning), or the package was reloaded since the build (a reload drops the bindings until the next run). Jump to **Debugging a no-op build**.\n\nThis file is the core. The details live beside it, one topic per file, and each is a prompt of its own:\n\n- `reference/annotation.md` - the `#@ persist` tag: what can carry it, the name rules, identity and reuse, readers of a persisted source, and every option with where it is explained.\n- `reference/policy.md` - the `publisher.json` block: `scope`, `queryMetadata`, and where the rules are enforced.\n- `reference/storage.md` - building into a **storage destination** (`storage=`), `partition=`, where a given may sit, which sources that tier refuses, and how a stored table is served.\n- `reference/chaining.md` - one stored source built **over another stored source**: what makes it reuse the parent's table or recompute from the warehouse, and how to read which happened.\n- `reference/refresh.md` - keeping a table current: `freshness`, `schedule`, and `refresh=\"incremental\"`.\n- `reference/access-filter.md` - persisting a `#(access_filter)`-gated source, and the bound on a stale access decision.\n- `reference/tuning.md` - reading the materialization history to decide what to persist, stop persisting, or reschedule (recommendations only).\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on every `.malloy` file that declares a persist source.** Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why it matters:** the build plan walks only the models that carry the flag, and skips every other file **without a warning**. The flag is a property of the model being *walked*, not of the file that declares the annotation: a flagged model builds every persist source its walk reaches, including ones declared in a flagless file it imports, and a flagless file's persist sources build only when some flagged model imports it. So a `#@ persist` in a flagless file that nothing flagged imports is silently never built, while adding a flagged entry model that imports it starts building tables that were inert - neither state warns. Put the flag in the declaring file itself: that is the one placement whose effect does not depend on what else imports it. A file with no persist source of its own does not need the flag; adding it anyway is harmless. (`#@ preaggregate` rollups are the one exception: they build from an unflagged file, because they live in a model Publisher synthesizes.)\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a `-> { ... }` pipeline or a `conn.sql(\"...\")`, including one refined by a trailing `extend { ... }`. A `#@ persist` on a *plain* `extend` over a bare `conn.table(...)` is **refused, not ignored**: the package warns, and every run that covers it fails naming the source, so a whole-package run builds nothing until you fix it. Persist a query source instead (`source: x is raw -> { where: ...; select: * }`).\n   - **Quote the name, once.** A bare `name=my_table` fails the package load with `persist annotation name must be quoted`; a name quoted inside the quotes silently serves live. The target's container must already exist and be writable by the connection.\n   - **The name is not the identity.** A persist source is identified by a content address of its connection, SQL and partition layout, so a rebuild with unchanged logic reuses the table and two sources with the same body share one. An `extend` of a persisted source reads the same table and is not a second build target.\n\n   The name rules, bound arguments, `#@ -persist`, and every option (`storage=`, `partition=`, `refresh=`, `freshness.*`, `queryMetadata.*`) are in `reference/annotation.md`.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"queryMetadata\": { \"team\": \"finance\" },   // root level: tags every statement a build OR a query issues\n     \"materialization\": {\n       \"scope\": \"package\",                      // default; \"version\" = each published version owns its own tables\n       \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" }\n     }\n   }\n   ```\n   - **`scope`** is a package-level contract about sharing tables across versions; on a standalone server it changes nothing except gating `schedule`. Its deprecated root-level home still works, but the two homes disagreeing **fails the load**.\n   - **`freshness`** is the objective a **hosted control plane** enforces by refreshing the table; a **standalone** Publisher does not act on it at all. `fail` behaves like `live` today; only `stale_ok` keeps serving a stale table.\n   - **`queryMetadata`** tags every build statement and every query for the backend's cost attribution. Root level; the home inside `materialization` is deprecated. Observability only.\n   - **`schedule`** is a 5-field UTC cron that **requires `scope: \"version\"`** and **excludes `freshness`**; it fires only on a standalone server whose scheduler is enabled.\n\n   The coherence rules are rejected at publish and at a policy edit, warned at load, and skipped by the scheduler. `reference/policy.md` has `scope` and `queryMetadata` in full; `reference/refresh.md` has `freshness` and `schedule`.\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --environment <env> --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed. Two flags to know: `--force-refresh` builds every source even when its content address is unchanged (which is what every scheduler fire does), and `--reseed` is the only thing that rebuilds an incremental source from scratch. `--wait` gives up after **120 s by default** (`--timeout <s>`) and exits non-zero while the build keeps running; `malloy-pub get materialization <id>` then shows where it got to.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nTwo mechanics either way. **One build at a time per package**: a second run requested while one is active is rejected with a 409 conflict rather than queued, and the scheduler skips a fire while a run is active. **A build can be partial**: `sourceNames` on the request builds only the sources it names and leaves the others exactly as they were (live if never built), which is how to add one source to a package without rebuilding the rest.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Confirming it worked\n\nA green run is not proof, and neither is a fast query (on small data a live recompute is fast too). Check what the server says it serves:\n\n1. **The run lists your source, and did not lose it.** A run's detail (`malloy-pub get materialization <id> --environment <env> --package <pkg>`) carries `manifest.entries`, a map keyed by `sourceEntityId` whose values name each built or reused source (`sourceName`) and its physical table. A run can finish `MANIFEST_FILE_READY` having built nothing (every source skipped for a missing flag, or carried forward unchanged) **or having lost a source**: a source that failed is absent from `entries` and present in `manifest.failures` with its error, and a source the eligibility gate refused is in `metadata.refusedSources`, while the run still reads ready. `metadata.sourcesBuilt` / `sourcesReused` / `sourcesFailed` / `sourcesRefused` are the counts. For a source built over another stored source, the entry's `upstreamReuse` says whether it read the parent's table (`reused`) or recomputed it (`recomputed`, with `upstreamRecomputeReason`) - see `reference/chaining.md`.\n2. **The package is bound to those tables.** On a standalone Publisher the package's details report `manifestBindingStatus` and `manifestEntryCount`. The count covers tables built in the source's own warehouse only: it reads `bound` with a nonzero count when such a table is bound and `unbound` otherwise. A `storage=` table never counts there - it appears under the package's `storageServeBindings`, so a package whose only persist sources are `storage=` reads `unbound` while serving from the store. `live_fallback` is a hosted-only status: a configured `manifestLocation` could not be fetched or bound, and the previously bound manifest (if any) stays in force.\n3. **The query names the table.** A query's full result (the REST query response without `compactJson`) carries the SQL it ran. Served from the table, its `FROM` names the physical table; computed live, it carries the source's own SQL instead. This is the per-query proof.\n4. **A reload unbinds them.** Reloading the package - including the automatic reload when a watched file is saved - drops the binding, and queries compute live until the next materialization run, which reuses unchanged tables and binds them again. (A server *restart* re-establishes the bindings from its store; a reload does not.) After editing a model, run a build before you trust a timing.\n\nA query response's `servedFrom` field does not answer this question for a table built in the source's own connection: it reports only the `storage=` tier's outcome (`storage` when that tier answered, `live_fallback` when it degraded to a live recompute) and is empty otherwise, served from a colocated table or not. Do not read `live_fallback` here as a signal about a colocated table.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run, and for a ready run with `failures`.\n1. **Did the run fail, or lose this source?** A `FAILED` run's `error` names what stopped it. A ready run's `manifest.failures` names each source that did not build, and `metadata.refusedSources` each one the gate refused, with the reason to act on. The package's details flag some of these before any run: an unbuildable persist source appears in its warnings, and its `buildPlan` lists the sources a run will build and the ones it will refuse. A `#@ persist` on a non-persistable source fails every run that covers it, naming the source.\n2. **The declaring model is missing the persistence flag** and nothing flagged imports it: skipped without any warning, the run succeeds, the source is simply absent from what was built. Add the flag to the declaring file.\n3. **An unquoted persist name** - a bare `name=foo` **always** stops the load with `persist annotation name must be quoted`. (If you got *no* error at all, it isn't this.)\n4. **The package was reloaded** after the build (a saved file under watch mode counts). The tables exist and the run reads ready; the binding is gone. Run a build. And if a watched edit failed to compile, the server keeps serving the **old** model and its binding, and records the error under `staleCompileErrors` on `/status` - so \"still bound\" after an edit can mean the edit never took.\n5. **A `storage=` source serves live while the store is switched off.** With `PERSIST_STORAGE_MODE=off` a `storage=` source is skipped by the build and served live, with no error; it is never built into the warehouse instead. See `reference/storage.md`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\nIf **even this** doesn't build after a real run, the problem is package-wide: read the run's error (a non-persistable source anywhere fails the whole run and names it), then check that the connection can write the target dataset. If the smoke source builds and your real one doesn't, your real source is the problem: its file's flag, or its entry in `failures` / `refusedSources`. Delete the smoke file and drop its table afterward (`malloy-pub delete materialization <id> --drop-tables`).\n\n## Gotchas\n\n- **Flag every file that declares a persist source** - a flagless model nothing flagged imports is skipped without a warning.\n- **A `#@ persist` on a plain `extend` over `conn.table(...)` fails the run** - every run that covers it, naming the source.\n- **A tag doesn't build** - only an explicit run, the standalone scheduler, or a hosted control plane does.\n- **Quote the name, once** - bare stops the load; double-quoted silently serves live.\n- **A ready run can still have lost a source** - read `manifest.failures` and `metadata.refusedSources`.\n- **A reload unbinds; a restart rebinds** - run a build before trusting a timing.\n- **Reuse is by content address, not by name** - `--force-refresh` and every scheduler fire defeat it.\n- **Removing a persist source does not drop its table** - `delete materialization --drop-tables` drops what no other ready run names, and serving reverts to live in place.\n- **A given inside the persisted query is refused; in the source's own `extend` block it is applied per caller** - `reference/storage.md`.\n- **A join to a non-persisted source, or a nested column, serves live for the queries that touch it** - the source's own fields still serve from the table.\n- **A `#(access_filter)`-gated source freezes its gating column when persisted** - `storage=` and `#@ preaggregate` refuse it outright; `reference/access-filter.md`.\n\n## Reference files over MCP\n\nThis skill's `reference/` files are served as separate prompts, one per file, fetched only when you ask for them. Where the text above says to read `reference/<name>.md`, get the prompt named `malloy-materialization/<name>` instead.\n\nAvailable: access-filter, annotation, chaining, policy, refresh, storage, tuning."
+    },
+    {
+      name: "malloy-materialization/access-filter",
+      description: "Persisting a `#(access_filter)`-gated source. Reference detail for the malloy-materialization skill.",
+      body: "# Persisting a `#(access_filter)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be careful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, naming it. The build skips a refused source, records it on the run (`metadata.refusedSources`), and builds the rest of the package. A run fails on a refusal only when every authored source it targeted was refused, or when `sourceNames` named this one; a refused rollup never fails it. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage destination and serves from there, rather than building in the source's own connection; `#@ preaggregate` stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's **own row filter**. The gate is found through the import -> rename -> `query_source` -> `extend` chain, so a gate the persisted source did not declare itself still counts, and an inherited gate is admitted like a declared one. It is refused in exactly two cases: the gate is reached only through a join, or it does not classify as a row filter at all.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and the gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered. What freezes is the **column the gate filters on**. A row whose access decision changes - it changes owner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access decision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what needs a bound is how long a *stale* decision can survive. Of the three controls that look like that bound, only the first is:\n\n- **`freshness.window=\"24h\"` with `freshness.fallback=\"live\"` is the bound** (`reference/refresh.md`). The serve path re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details decide whether you actually get that. **`fallback` must not be `stale_ok`**: under `stale_ok` a stale artifact keeps being served, which voids the bound (`live` and `fail` both bound it today). Window and fallback resolve *independently, per field*, across the source, model-file and package layers, so a package-level `stale_ok` silently defeats a window you set on the source. Prefer the **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting it package-wide forces every other persisted source to recompute once stale too. And **a content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness policy. A host that folds the siblings' policies keeps the tightest window any of them declares - a sibling declaring nothing cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses - and keeps the bounding fallback, so a sibling's `stale_ok` cannot void your bound either. If two sources need genuinely different windows, give them genuinely different SQL.\n\n  Both of those are properties of the **host** that assembles the manifest, not of the annotation: the publisher reports freshness per source and does no folding itself. Where the host does not fold, which sibling's policy reaches the wire is unspecified; a host that folds does so when it next publishes a version's manifest, not retroactively to manifests already distributed - so you can declare the window correctly and not have it in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a control plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the table with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone deployment the declared window is inert unless something binds the manifest through the API with those fields - which leaves a rebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source unpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what staleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no bound on how long a revoked row keeps being served."
+    },
+    {
+      name: "malloy-materialization/annotation",
+      description: "The `#@ persist` annotation. Reference detail for the malloy-materialization skill.",
+      body: '# The `#@ persist` annotation\n\n```malloy\n#@ persist name="my_dataset.my_table"\nsource: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n```\n\n## What can carry it\n\n**Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql("...")`, including one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)` (a filtered pass-through). A `#@ persist` on such a source is **refused, not ignored**: the package carries a warning naming the source, and any materialization run that covers it fails with `annotated \'#@ persist\' but were not recognized as a materializable source`, so a whole-package run builds nothing until you fix it. Persist a query source instead (`source: x is raw -> { where: ...; select: * }`), or remove the annotation.\n\n**A parameterized source must have its argument bound.** `#@ persist` on `high_value(min_amount is 500) -> { ... }` builds one fixed relation; on the free template `high_value -> { ... }` it is refused (`free_parameter`), since there is no single relation to store. A `given` the persisted *query* reads is refused the same way (`given_in_persisted_query`); a given in the source\'s own `extend` block is applied per caller at read. `reference/storage.md` says where a given may sit.\n\n## The name\n\n- **Quote it, once.** `name="my_table"`, or a path `name="dataset.table"` / `name="project.dataset.table"`. A **bare** `name=my_table` fails the package **load** - so every publish and every restart - with `persist annotation name must be quoted`; it never silently no-ops.\n- **Plain identifier segments only**: dot-separated segments of letters, digits, underscores and hyphens. A quoted name holding a space, quote, semicolon or backtick is rejected with its own message, because the name is inlined into DDL.\n- **Never quote inside the quotes.** `name="\\"Quoted Tbl\\""` reaches the build with literal quote characters: the table is created with them in its name, the serve side looks for the unquoted one, and the source silently serves live after a wasted build.\n- **One name per table.** Two distinct persist sources that resolve to the same `name=` in the same destination clobber each other (the second build overwrites the first, and dropping one drops the other\'s). Publisher warns on the package at load and, where `PERSIST_COLLISION_ENFORCE` is set, rejects the publish. A shared *body* is fine (it shares one table by content address); a shared name over different bodies is not.\n- **The target\'s container must exist.** `name="analytics.daily"` puts the table in `analytics`, which the connection must be able to write and which Publisher does not create. The source can *read* anything the connection can read.\n- **The name is not the identity.** In a standalone Publisher `name=` is the physical table, rebuilt in place; a hosted deployment may assign the physical name itself. A `storage=` destination takes a single undotted name, since placement there is the destination\'s.\n\n## Identity and reuse\n\nA persist source\'s identity is its **`sourceEntityId`**: a content address of its connection, its canonical SQL and its `partition=` layout. Not the name, not the file, not the package version. So:\n\n- **A rebuild with unchanged persist logic reuses the existing table** (skip-if-unchanged); changing the SQL or the partition layout builds fresh under a new address. `--force-refresh` and every scheduler fire defeat the skip; an incremental source is exempt from it and advances by its boundary instead (`reference/refresh.md`).\n- **Two sources whose bodies compile to the same SQL share one table**, however they are named, and share one freshness policy and one incremental boundary with it.\n- **A republish by itself builds nothing on a standalone server**; reuse is a property of the next *run*.\n\n## Readers of a persisted source\n\n- **An `extend` of a persisted source reads the stored table** and is not a build target of its own: it inherits the parent\'s tag and the same address, so the two names are one table. To make a distinct non-persisted intermediate, pipe first: `base -> { select: * } extend { ... }`.\n- **A query over a persisted source** (`orders is _orders_fact -> { select: * }`, the private-fact / public-wrapper idiom) reads the table too, with the fact\'s `extend`-block filters applied per caller, and is not a build target.\n- **`#@ -persist`** on an extension opts it out: it recomputes from raw on every query and never reads the table. Reach for it when a reader must not see stale rows, and remember it forgoes exactly the work persistence was there to save.\n- Which queries over a reader route to the table is decided per query - a join to a non-persisted source or a nested column serves live for the queries that touch it (`reference/storage.md`).\n\n## Every option\n\n| On the tag | What it does | Where it is explained |\n|---|---|---|\n| `name="..."` | the target table (required) | above |\n| `storage=<destination>` | build into a storage destination and serve from it | `reference/storage.md` |\n| `partition="a,b"` | lay the stored table out by those columns (`storage=` only) | `reference/storage.md` |\n| `refresh="incremental" watermark="col" [merge_key="a,b"]` | refresh by bounded delta instead of a full rebuild | `reference/refresh.md` |\n| `freshness.window="24h" freshness.fallback="live"` | per-source freshness objective (hosted deployments) | `reference/refresh.md` |\n| `#@ queryMetadata.<key>="v"` (sibling line) | per-source cost-attribution properties | `reference/policy.md` |\n| `#@ -persist` (on a reader) | opt a reader out of the table | above |\n| `#@ preaggregate grain="..."` (on a measure) | store a derived rollup | `reference/storage.md` |\n\nAn unrecognized key on the tag is a warning on the package. The retired per-source `sharing=` and `schedule=` are rejected at publish: scope and schedule are declared once for the package (`reference/policy.md`).'
+    },
+    {
+      name: "malloy-materialization/chaining",
+      description: "Chaining: one stored source built over another. Reference detail for the malloy-materialization skill.",
+      body: `# Chaining: one stored source built over another
+
+A \`storage=\` source may read another \`storage=\` source. When both land in the same destination, the downstream is built by **reading the upstream's stored table** in the destination's own engine - the upstream is never re-scanned from the warehouse - and the chain is consistent by construction: the child is a pure function of the parent's stored rows. Chaining is what turns a package of independent rebuilds into one that touches the warehouse only for its roots.
+
+\`\`\`malloy
+#@ persist name="daily_orders" storage=lake
+source: daily_orders is orders -> { group_by: order_date; aggregate: total_amount is amount.sum() }
+
+#@ persist name="monthly_orders" storage=lake
+source: monthly_orders is daily_orders -> { group_by: order_month is order_date.month; aggregate: monthly_total is total_amount.sum() }
+\`\`\`
+
+The downstream need not name the upstream directly. Non-persisted sources between them - a \`select: *\` wrapper, an \`extend\` that adds a dimension, a named source over an inline \`(parent extend { ... }) -> { ... }\`, a query over the upstream, a parent reached only through an \`import\` - are carried into the build in dependency order, under the \`##!\` flags of the files that declare them, and declaring the author model's \`given:\`s, so the downstream still reads the upstream's table through them.
+
+## The four rules that decide reuse
+
+1. **Declare the stored roots before the sources that read them.** Malloy resolves names in file order. A rollup rewired to read a stored root that is declared further down fails to compile with \`Reference to undefined object\`, and so does everything after it. When you chain an existing package onto its roots, move the roots up.
+2. **An extension of a stored source is the same table, not a new one.** \`rollup_base is daily_orders extend { ... }\` inherits the parent's persist tag and the same content address, so the publisher treats the two names as one build target. To make a distinct non-persisted intermediate, pipe through a query first: \`rollup_base is daily_orders -> { select: * } extend { ... }\`. (A host that checks persist names for collisions can refuse the bare \`extend\` outright, since two names now point at one table.)
+3. **Every join on the path must land on a stored table.** An intermediate that joins a source nothing materializes - a \`join_cross: sp is species_live\` where only \`species_base\` is stored - reaches the warehouse, and the whole downstream is recomputed from the warehouse with the upstream inlined. Point the join at the stored sibling instead.
+4. **A chained build runs in the destination's engine, under its memory bound.** The rows are read from the lake and rolled up in DuckDB with the build session's \`memory_limit\`. A grouped rollup fits; a self-join or a wide hash aggregate over a large fact may not, and fails with DuckDB's \`Out of Memory Error\` on every attempt. Such a source belongs in the warehouse: keep its input unpersisted so it is built there and only its result is copied in. Placement is a cost decision the build does not make for you.
+
+## What the build does when it cannot reuse
+
+If the downstream cannot be built over the stored tables - it reaches the warehouse through an intermediate (rule 3), or reads a field on the parent that is not a stored column - the publisher **recomputes the upstream from the warehouse**, inlining it, and the build succeeds. Under \`strictUpstreams\` (what an orchestrating host sets on every build) this is the **one** recompute strict permits, because no build over the stored tables exists for that shape. Strict still refuses what it exists to refuse: a persisted upstream the build neither materialized nor was handed by reference, one whose table lives in a *different* destination, and a source over stored upstreams that the build could not carry (a refinement the ladder below drops). A refusal names the upstream and where it lives.
+
+A chained build re-declares on each parent's binding what the parent's \`extend\` block adds - its \`where:\`, dimensions, measures, joins and views - and thins them when they do not compile in the destination, in tiers: a view first, then a join, then a dimension or measure that reads a join the destination cannot bind. **The \`where:\` is never thinned**: a stored source declared \`extend { where: total > 100 }\` holds every row and is filtered when read, so a child built over it without that filter would sum the unfiltered rows.
+
+## Reading which happened
+
+Every manifest entry for a source that reads a persisted upstream carries:
+
+- \`upstreamReuse: "reused"\` - every persisted upstream was read from its table.
+- \`upstreamReuse: "recomputed"\` - at least one was recomputed from its definition; \`upstreamRecomputeReason\` names which and why (for example, which intermediate reached the warehouse through which join).
+
+It is reported on both tiers, since a table built in the source's own warehouse also inlines a \`storage=\` upstream it cannot read there. The field is not a location and never names a table. A recomputed source's rows reflect the warehouse at *that build's* time rather than the upstream's snapshot, so the two can disagree when their refresh cadences differ - this field is how to tell.
+
+The \`publisher_storage_chained_build_total{outcome}\` counter reports the path each chained build took: \`parent_reuse\`, \`inline_fallback\` (non-strict recompute), \`strict_shape_fallback\` (the strict-permitted recompute), \`strict_refused\`, \`infra_failure\` (the destination was unreachable or the build died in it - kept distinct so a store outage or an out-of-memory is not read as an un-carriable shape).
+
+## Three limits to know
+
+- A refresh of a chained \`refresh="incremental"\` source always rebuilds in full (\`chained_storage\` reason): its parent's delta can restate rows below the child's own frontier where no delta of the child's would revisit them.
+- A \`#@ persist\` without \`storage=\` (built in the source's warehouse) never substitutes a \`storage=\` upstream's table, because the warehouse cannot see the lake; it inlines the upstream and reports \`recomputed\`.
+- A chained build carries the intermediates' *text*, written for the source warehouse, into the destination's dialect. A construct the destination lacks fails the carry, and under strict that is a refusal rather than a recompute.`
+    },
+    {
+      name: "malloy-materialization/policy",
+      description: "The package policy: `scope`, `queryMetadata`, and where the rules are enforced. Reference detail for the malloy-materialization skill.",
+      body: '# The package policy: `scope`, `queryMetadata`, and where the rules are enforced\n\n```jsonc\n{\n  "name": "my-package",\n  "queryMetadata": { "team": "finance" },   // root level\n  "materialization": {\n    "scope": "package",                      // default; or "version"\n    "freshness": { "window": "24h", "fallback": "live" },\n    "schedule": "0 6 * * *"                  // never together with freshness\n  }\n}\n```\n\n`freshness` and `schedule` are explained in `reference/refresh.md`. This file covers the rest of the block and the gate that checks it.\n\n## `scope`\n\n`package` (default) or `version`. It is a package-level contract about whether a persisted table may be shared across published versions: under `package` an unchanged source\'s table is reused by every version that computes the same SQL; under `version` each published version owns its own tables. There is no per-source scope and no per-source schedule; both are declared once for the package.\n\nTwo things to know about it on a standalone server:\n\n- **It changes nothing there except gating `schedule`.** A standalone auto-run keys tables by content address regardless of scope; true per-version tables are produced when a hosted control plane assigns versioned build targets. Standalone is the place to exercise the *policy* rules, not per-version fan-out.\n- **A root-level `scope` is the deprecated home** and still works, with a warning. Whenever the server rewrites the manifest (any package PATCH, including a description-only one) it writes **both** homes with the same value, so the root key reappears after an API edit. **The two homes holding different values is not a warning: the package fails to load**, disappears from the server, and says so only in `/status` `loadErrors`. Editing by hand, change `materialization.scope` and delete the root copy; never edit the root one.\n\n## `queryMetadata`\n\nA bag of up to 20 string properties attached to **every statement a build issues and every query against the source**, for the backend\'s own cost attribution: Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere. Publisher adds its own context on top (`class=materialize` with the package, source, trigger and run id on a build; `class=interactive` on a served query), and drops a model-side declaration that reuses one of those server-owned names.\n\n- **Declare it at the root of `publisher.json`.** The older home inside `materialization` still works but emits a deprecation warning on every load and publish.\n- **Three layers, most specific wins:** a sibling line on the source (`#@ queryMetadata.team="risk"` above the `source:`), a model-file line (`## queryMetadata.team="risk"`), then the package block.\n- **Observability only.** It never changes what gets built, is not part of the content address, and problems with it (a bad key, an over-long value, the deprecated home) are warnings, never rejections.\n\nSee `docs/query-metadata.md` for the per-backend rendering.\n\n## Where the rules are enforced\n\nThe scope / schedule / freshness coherence rules - a single package-level scope, `schedule` requires `scope: "version"`, `schedule` and `freshness` (including a per-source `freshness.*`) are mutually exclusive, a valid 5-field UTC cron - are checked in four places:\n\n| Where | Outcome |\n|---|---|\n| publish | rejected (400) |\n| a package edit that touches the policy | rejected (400); an edit that does not touch it is not re-checked |\n| package load | warned; the package still serves |\n| the scheduler | an offending package is skipped |\n\nOnly those coherence rules are gates. `queryMetadata` findings, the deprecated homes, and an unrecognized `#@ persist` key are advisory. An invalid `refresh="incremental"` declaration is the other way round: it **fails the load** outright (`reference/refresh.md`). The `malloy-pub schedule` commands share the publish gate, so a rejection from them means the change was unsafe - and `schedule set` replaces the whole `materialization` block, dropping a freshness policy it finds there.'
+    },
+    {
+      name: "malloy-materialization/refresh",
+      description: "Keeping a table current: freshness, schedule, incremental refresh. Reference detail for the malloy-materialization skill.",
+      body: '# Keeping a table current: freshness, schedule, incremental refresh\n\nA persisted table is a snapshot. Three declarations govern when it is rebuilt and what a reader gets in the meantime; which of them does anything depends on who drives the builds.\n\n## `freshness`: the objective tier\n\n```malloy\n#@ persist name="daily_orders" freshness.window="24h" freshness.fallback="live"\n```\n\nor package-wide in `publisher.json` under `materialization.freshness`, or per model file with `## materialization.freshness.window="24h"`. The three layers resolve **per field** - source, then model file, then package - so a source that declares only `window` takes its `fallback` from the package. That is how a package-level `stale_ok` silently defeats a window set on one source.\n\n- `window` is how old a table may be before it is stale. `fallback` says what a query over a stale table does: `live` and `fail` both skip the table and compute live (`fail` is reserved to error later; today it behaves as `live`); `stale_ok` keeps serving the stale table.\n- A **hosted control plane** enforces the objective: it refreshes the table to stay within the window and stamps each manifest entry with `dataAsOf` and the window, and the serve path re-checks freshness **per query**, so a table that ages past its window drops out of the serving set whether or not a rebuild ever lands.\n- A **standalone** Publisher does not act on it: it binds what it just built with no `dataAsOf` and no window, and an entry carrying no window never ages out. The declaration is inert there unless something binds the manifest through the API with those fields set. For a standalone server, the only refresh is a run or the scheduler below.\n- Two sources whose bodies compile to the same SQL share one table, and a host that folds their policies keeps the tightest window and the bounding fallback for both. If two sources need different windows, give them different SQL.\n\n## `schedule`: the power tier\n\n```jsonc\n{ "name": "orders", "materialization": { "scope": "version", "schedule": "0 6 * * *" } }\n```\n\n- A 5-field UTC cron; `L`, `W`, `#`, `?` are rejected at publish. It **requires `scope: "version"`** and is **mutually exclusive with `freshness`** - including a per-source `freshness.*`.\n- It fires only on a standalone Publisher whose scheduler is enabled with `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER`; without that, a schedule is inert. The scheduler sweeps loaded packages, skips one whose policy is invalid or that a control plane manages (`manifestLocation` set), fires at most a bounded number per tick, fires once to catch up after downtime, and skips a fire while a run is already active.\n- **Every scheduled fire is a forced run** (`forceRefresh: true`): every source is rebuilt whether or not its content address changed. A scheduled run therefore never reports reused sources, and a cron that is too frequent shows as build time, not as reuse.\n- `malloy-pub schedule set "<cron>"` writes the schedule and sets `scope: version` in one PATCH; it **replaces the whole `materialization` block**, dropping an existing `freshness` policy (it says so). `schedule clear` removes the cron and leaves `scope: version` in place; moving back to `package` scope is an edit to `publisher.json`.\n\n## `refresh="incremental"`: bounded deltas\n\n```malloy\n#@ persist name="daily_orders" refresh="incremental" watermark="order_date"\nsource: daily_orders is orders -> { group_by: order_date; aggregate: revenue is amount.sum() }\n```\n\n- `watermark=` names one of the source\'s own output columns - real, orderable, non-aggregate. Publisher records how far the table is materialized (`covered_through`) and each refresh recomputes only `[covered_through, frontier)`, half-open, so the frontier value itself is left for the next run. A `date`/`timestamp` watermark takes the run\'s start time as the frontier; a numeric or string one reads `max(watermark)` from the source. Two refreshes within the same day of a date-watermarked source produce an empty range and skip - correct, and it looks like nothing happened.\n- Add `merge_key="col,..."` when a row can be **restated** with a new watermark value; the delta is then applied as a `MERGE` on that identity instead of a delete-and-reinsert of the range. On a caller-scoped source (a given or gate in the `extend` block) the table holds every caller\'s rows, so the match also carries the scope columns - declare the key as you would for an unscoped source. A scope term that reaches through a join is refused (`merge_key_scope_unresolved`) rather than scoped partially.\n- **Coverage belongs to the table.** Moving a source between tiers (adding or removing `storage=`), renaming its physical table, or changing its SQL re-seeds it in full; the boundary is measured against the table it was recorded on.\n- **An invalid declaration fails the package load**, not just the publish: `watermark=` without `refresh="incremental"`, `merge_key=` without a watermark, a watermark that names no materialized column or an aggregate, a `calculate:` field, or an unsupported dialect. Postgres, BigQuery and Snowflake sources only (the dialect that has to express the range); a DuckDB source has to say `refresh="full"`.\n- **`forceRefresh` never re-seeds**; it only defeats skip-if-unchanged, and an incremental source is exempt from that anyway. Ask for a full rebuild with `reseed` (`malloy-pub materialize --reseed`, or per source on the build instruction). Keeping them apart is what lets a schedule drive deltas at all, since every scheduled fire is forced.\n- Each manifest entry reports what the run **did**: `refresh: delta | full | none`, with `ledger.coveredThrough` as the boundary now in force. Every fallback - no recorded boundary, a boundary measured on a different table or watermark, drifted columns, `MERGE` on a Postgres older than 15 - rebuilds in full and succeeds, so a source quietly rebuilding on every run looks exactly like one advancing unless you read `refresh`. The reason codes (`forced`, `no_boundary`, `lineage_changed`, `table_renamed`, `merge_unsupported`, `table_unreadable`, `table_emptied`, `shape_mismatch`, `ledger_unreadable`, `chained_storage`, `frontier_unreadable`, `not_advanced`) say why.\n- Two sources that compile to identical SQL share one table **and one boundary**; if their incremental declarations differ, neither ever advances. Publisher warns and names both.\n- `storage=` is supported: the warehouse computes the bounded range and the DML lands in the destination. A **chained** stored source always rebuilds in full (`chained_storage`): see `reference/chaining.md`.\n- The boundary lives in the publisher\'s own per-process store. A host that spreads a package\'s refreshes over several workers holds the ledger itself: it stores each entry\'s `ledger` object and sends them back as `buildInstructions.ledger` on the next run. Standalone, the store is the publisher\'s, always.\n\n## Which refresh a gated source needs\n\nA `#(access_filter)`-gated source that is persisted freezes the column its gate filters on, so its "refresh" is also the bound on how long a revoked row keeps being served. Only a freshness window with `fallback: live` on a hosted deployment bounds that; a cron alone does not, and an incremental delta never re-reads a row whose watermark did not move. See `reference/access-filter.md`.'
+    },
+    {
+      name: "malloy-materialization/storage",
+      description: "The storage tier: `storage=`, `partition=`, and how a stored table is served. Reference detail for the malloy-materialization skill.",
+      body: '# The storage tier: `storage=`, `partition=`, and how a stored table is served\n\nBy default a persist source is built **in its own connection** (colocated): the warehouse runs the query and the table lands beside the data. `#@ persist name="..." storage=<destination>` builds it into a **storage destination** the environment declares instead - today a DuckLake catalog over local or object storage - and serves it from there, so a query served from the table never runs in the warehouse at all.\n\n```malloy\n#@ persist name="daily_orders" storage=lake\nsource: daily_orders is orders -> { group_by: order_date; aggregate: total is amount.sum() }\n```\n\n- `storage=` names a **storage destination**, never a connection. Destinations are declared on the environment (`storageDestinations: [{ "name": "lake", "type": "ducklake", "ducklakeConnection": { "catalog": {...}, "storage": {...} } }]`, by config or by environment PATCH); a build naming one the environment does not hold fails the run with a `DestinationNotFoundError`. Reads of the environment report a destination\'s name and type only, never its config.\n- The build is a native **query passthrough**: the source warehouse (Postgres, BigQuery or Snowflake) computes the query and only the result crosses into the store, through DuckDB. A source on a DuckDB connection is refused for `storage=` (there is nothing to push down). A `#(access_filter)`-gated source is always refused on this tier (`reference/access-filter.md`).\n- The tier is gated by `PERSIST_STORAGE_MODE` (`off` / `write-only` / `on`). Under `off` a `storage=` source is **skipped by the build and served live, with no error** - it is never built colocated instead. Under `write-only` it builds but is still served live, and the package warns. If a `storage=` source "did nothing", check the mode before the model.\n- A package may name several destinations; each source routes to its own.\n- `partition=` requires `storage=` (refused as `partition_without_storage` otherwise) and every named column must be one of the source\'s public output columns.\n\n## Where a given may sit\n\nA persisted table is built once, with no caller present, so a `given` has to be somewhere the build never reads it.\n\n- **Inside the persisted query - refused.** `scoped_rollup is orders -> { where: region = $REGION; group_by: ... }` has only the declaration default available at build time, so the table would hold one caller\'s slice and nothing at read would re-apply the filter. The build refuses it (`given_in_persisted_query`). The same applies when the given is *inherited*: a query over a source whose `extend` block carries the term reads that filter into its SQL, and is refused just the same - what decides is whether the **build** would substitute a value, not which line the author typed it on.\n- **In the persist source\'s own `extend` block - honoured per caller.** `#@ persist ... storage=lake` on `fact is raw -> { select: * } extend { where: org_id = $ORG_ID }` strips the term from the build, stores every caller\'s rows in one table, and re-applies the term per caller at read. One artifact serves every tenant, and a term a caller writes on an extension of it still scopes what is read. Partition on the scoped column (`partition="org_id"`) so each caller\'s reads touch one partition.\n\nA `#(access_filter)` gate is the same shape with a different source of truth; see `reference/access-filter.md`.\n\n## `partition=`: laying the table out\n\n`#@ persist ... storage=lake partition="org_id"` writes the stored table as one file set per distinct value, so an equality term on that column (`=`, `in`) reads only the files it names; `partition="org_id,day"` nests them in the order given. It is a **layout and carries no isolation**: every filter is re-applied at read whether or not its column is partitioned, so a partition list that omits the column a query filters by costs a full scan, never a wrong answer.\n\nUse it for the column the reads filter by. A rollup carrying every key at a fine grain (every species, every tenant, every product) that each query filters to one key is read **whole** per query without a partition on that key - that is what makes a pre-aggregated table answer in seconds instead of milliseconds. Partition on the filter key and read cost tracks one partition while build cost tracks the whole table. Nothing checks the column\'s cardinality for you: a partition per value across very many values is the many-small-files case, and that judgement is yours.\n\n`partition=` is part of the source\'s content address, so changing it rebuilds the table. A chained downstream of a partitioned parent reads the parent\'s table like any other.\n\n## Shaping a rollup for its reads\n\nA stored rollup pays at build time so that reads are cheap; the grain and the inputs decide whether they are.\n\n- **Grain to the reads, not to the data.** Group by exactly the columns the queries filter or group by. Every extra dimension multiplies the row count, and a rollup that approaches the fact table\'s size saves nothing.\n- **Bake the filters into the query.** A `where: is_valid = 1` inside the `-> { ... }` shrinks the table; the same clause in a trailing `extend { ... }` is re-applied at read and the table holds every row.\n- **Give a ratio both halves.** A percentage whose numerator is a rollup and whose denominator is a count over the fact table still scans the fact per query. Persist the denominator at its own grain too.\n- **List the columns a pass-through keeps.** `select: *` on a stored copy of a wide fact carries columns no reader uses; name the ones the downstream rollups read.\n\n## How a stored table is served\n\nA query against a package with bound `storage=` tables is compiled over a transient model in which each stored source is rebound to its table - a *virtual source* - under the `##!` flags of the files that declared the originals, with their `extend`-block refinements re-declared on top. A public wrapper over a stored source (`orders is _orders_fact -> { select: * }`, with or without `include { public: ... }`) is carried onto that shape verbatim and served from the table. A wrapper whose query or joins reach a source that is not materialized there - a warehouse table, or a stored source whose table is stale past its freshness window - is served **live** instead.\n\nRouting is decided per query, not per source, so one source can serve some queries from its table and others live:\n\n- **A join to a non-persisted source** serves live for the queries that traverse it; queries on the fact\'s own fields still route to the table. A join between two persisted sources routes both legs.\n- **A nested column** (`nest:`, a repeated record) is stored, but the serve shape carries it as opaque JSON, so every query that traverses it pays the live cost while the scalar columns serve from storage. The source looks fully materialized and is, for its flat fields.\n- **A refinement the shape cannot reproduce is thinned** (a view, then a join, then a dimension or measure reading a join); a **filter is never thinned** - a source whose filter the shape cannot reproduce loses the tier itself, and only itself.\n- A **stale** table (past its freshness window, with `fallback` `live` or `fail`) leaves the serving set, and a run-time store failure under `fallback: live` degrades that query to live.\n\nWhat the server reports:\n\n- `QueryResult.servedFrom`: `storage` when the tier answered, `live_fallback` when the tier was bound but a run-time store failure degraded the query to a live recompute, and empty for everything else (a colocated hit, no binding, or an ineligible query - three different things).\n- The package\'s `storageServeBindings` lists the stored tables the package is currently bound to, with each entry\'s origin (`persist` or `preaggregate`). `manifestEntryCount` does not count them.\n- `publisher_storage_serve_routing_total{outcome=storage|live_fallback|runtime_live_fallback|blocked_by_row_level_gate, origin}` counts routed queries; `publisher_storage_serve_shape_tier_drop_total{tier}` counts refinements the serve shape had to thin.\n- A query\'s full result carries the SQL it ran; served from the store, its `FROM` names the destination and the physical table.\n\n## Pre-aggregation uses the same tier\n\n`#@ preaggregate grain="category, order_day"` on a **measure** stores a rollup Publisher derives at that grain and routes to it when a query groups by a subset of the grain; queries do not change. It builds through the same plan, manifest and scheduler as `#@ persist`, needs no `##!` flag, inherits the base source\'s freshness, and refuses a gated source. A rollup on a measure the published surface hides is not built; the package loads and warns. See `docs/preaggregation.md` for grain choice.'
     },
     {
       name: "malloy-materialization/tuning",
       description: "Tuning materializations for cost and performance. Reference detail for the malloy-materialization skill.",
-      body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n  ```bash\n  malloy-pub schedule view --environment <env> --package <pkg>\n  ```\n\n  This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n  ```bash\n  malloy-pub list materialization --environment <env> --package <pkg>\n  ```\n\n  Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n  ```bash\n  malloy-pub get materialization <id> --environment <env> --package <pkg>\n  ```\n\n  In the JSON, read:\n  - `metadata.durationMs`: how long the build took.\n  - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n  - `metadata.sourcesRefused` and `metadata.refusedSources`: sources the eligibility gate skipped. Each one serves live on every query, so a refused source the user expected to be persisted is a finding in its own right; its entry's `message` says what to change.\n  - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n  - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries a `#(access_filter)` gate, check which case it is first. A gate reached only through a `join_*`, or inherited from a base it cannot be attributed to, means the colocated persist is **refused** - the build skips it and it serves live - so do not recommend persisting it at all. Where the gate is provably the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `SKILL.md` in this skill, which also covers where that window does and does not bind).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n  ```bash\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n  ```bash\n  malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n  > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n  ```bash\n  malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg>   # 5-field UTC cron\n  malloy-pub schedule clear --environment <env> --package <pkg>\n  ```\n\n  `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
+      body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run and per-source timings, which sources were built vs reused, and for the storage tier how queries were routed) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, which to chain or partition, and how to schedule them. The Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"...\"` in the package's `.malloy` files, with their tier (`storage=` or not), `partition=`, and `refresh=`. Read the models (or `get_context` the package) to list them, and read the package's `buildPlan` for what a run will build and what it will refuse.\n- **Schedule + scope:**\n\n  ```bash\n  malloy-pub schedule view --environment <env> --package <pkg>\n  ```\n\n  This prints the cron (or `none`, meaning on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. It also says when a control plane manages the package (`manifestLocation` set): leave that package's cadence alone, the control plane refreshes it. And remember the standalone scheduler only runs with `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` set; a cron on a server without it is inert.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and per source what it did.\n\n- **For one package** (newest first; `--limit` / `--offset` page it):\n\n  ```bash\n  malloy-pub list materialization --environment <env> --package <pkg>\n  ```\n\n  Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n  ```bash\n  malloy-pub get materialization <id> --environment <env> --package <pkg>\n  ```\n\n  In the JSON, read:\n  - `metadata.durationMs`: how long the whole run took.\n  - `metadata.sourcesBuilt` / `sourcesReused` / `sourcesFailed`: how much work the run did. A run that reused most sources was cheap. **A `SCHEDULER` run never reuses**: every scheduled fire is forced, so its `sourcesReused` is 0 by construction; judge a scheduled run by its duration and its per-source entries instead. An incremental source is counted as built even when its refresh applied nothing.\n  - `metadata.sourcesRefused` and `metadata.refusedSources` (present only when a source was refused, and only on a run the publisher planned itself): sources the eligibility gate skipped. Each one serves live on every query, so a refused source the user expected to be persisted is a finding in its own right; its entry's `message` says what to change.\n  - `manifest.entries` - a map keyed by `sourceEntityId`. Each entry carries `sourceName`, `physicalTableName`, `realization`, and the per-source cost signals: `buildDurationMs`; `queryCostBytes` where the warehouse reports it; `refresh: delta | full | none` for an incremental source (a `full` on an unchanged source every run is a source that never advances); `upstreamReuse: reused | recomputed` for a chained source (a `recomputed` with `upstreamRecomputeReason` is a chain that went back to the warehouse).\n  - `manifest.failures`: sources the run lost while still finishing ready.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how long each source takes, what reuses) is the signal.\n\n- **For `storage=` tables, how queries were routed.** `publisher_storage_serve_routing_total{outcome, origin}` on the metrics endpoint counts queries answered from the store (`storage`) against ones that degraded (`live_fallback`, `runtime_live_fallback`), and each query response's `servedFrom` says it per query. Nothing equivalent exists for a table built in the source's own warehouse: there, \"rarely queried\" is a judgement from the model and the user's knowledge, not a measured metric. Say so when it matters.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"...\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries a `#(access_filter)` gate, check which case it is first. A gate reached only through a `join_*`, or not classifying as a row filter, means the colocated persist is **refused** - the build skips it and it serves live - so do not recommend persisting it at all. Where the gate is the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `access-filter.md`).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Chain candidate:** a `storage=` rollup whose intermediates re-read the warehouse on every build while a stored root already holds the rows. Rewire the intermediates onto the stored root (declaration order, a distinct `-> { select: * }` intermediate, joins onto stored siblings - `chaining.md`), and read `upstreamReuse` on the next run to confirm the chain holds. Keep a self-join or a wide hash aggregate over a large fact **in the warehouse**: chained, it builds in the destination's engine under its memory bound.\n- **Partition candidate:** a `storage=` rollup carrying every key at a fine grain that queries filter to one key - read whole per query. Recommend `partition=\"<key>\"` on the filter column (`storage.md`), noting the cardinality.\n- **Denominator candidate:** a ratio whose numerator is a rollup and whose denominator counts the fact table per query. Persist the denominator at its own grain.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If the build's `durationMs` approaches the interval, the cadence is too aggressive. If queries routinely read stale data, tighten it. For an incremental source, `refresh: delta` with small `buildDurationMs` per fire is the cheap steady state; `refresh: full` every fire means the boundary is being lost (read the reason).\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing it is step one; the scope itself is edited in `publisher.json` (the CLI does not move it back).\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts, per-entry fields) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n  ```bash\n  malloy-pub materialize --environment <env> --package <pkg> --wait --timeout 900\n  ```\n\n  (`--wait` gives up after 120 s by default and exits non-zero while the build continues; raise `--timeout` for a real build.)\n\n- **Remove a persist source:** delete the `#@ persist` annotation, rebuild, then drop the old run's tables:\n\n  ```bash\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  malloy-pub delete materialization <old-id> --environment <env> --package <pkg> --drop-tables\n  ```\n\n  > `--drop-tables` drops the physical tables in that run's manifest **except any a still-ready run also names** - with the stable names an auto-run assigns, a table the new run carried forward survives the delete. Deleting a run also re-derives the package's serve bindings from the latest remaining ready run (or clears them to serve live), so a delete does not break queries; it can leave a table behind. A removed source's table is therefore dropped only once no ready run names it - delete the old runs that do, or drop it in the warehouse yourself.\n\n- **Change the schedule / cadence:**\n\n  ```bash\n  malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg>   # 5-field UTC cron\n  malloy-pub schedule clear --environment <env> --package <pkg>\n  ```\n\n  `set` also sets `scope: version` (a schedule requires it) and **replaces the whole `materialization` block, dropping an existing `freshness` policy**; the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe. `clear` leaves `scope: version` in place.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, a chain that reports `reused`, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot measure reads of a table built in the source's own warehouse (only the storage tier reports routing), so \"rarely queried\" there is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
     },
     {
       name: "malloy-model",
@@ -396868,7 +397347,7 @@ Available: _concepts, cookbook-filter-context, cookbook-structure, cookbook-time
     {
       name: "malloy-powerbi-review/_concepts",
       description: "Power BI → Malloy Concept Mapping. Reference detail for the malloy-powerbi-review skill.",
-      body: "# Power BI → Malloy Concept Mapping\n\nReference table for translating Power BI (TMDL / DAX) constructs to Malloy. Referenced by multiple reference files.\n\n## Model Objects\n\n| Power BI (TMDL) | Malloy | Notes |\n|--------|--------|-------|\n| `table X` | `source:` | One source per table |\n| `column Y` | column reference, or `dimension:` when renamed or derived | Plain columns need no declaration |\n| `column Y` with `type: calculated` | `dimension:` with the translated expression | DAX evaluated at refresh; see `translate-measures.md` |\n| `measure M = <DAX>` | `measure:` | The translation is the hard part, not the syntax |\n| `partition P = m` with `mode: import` | `conn.table('schema.table')` | The M code names the real source table |\n| `partition P` with `mode: directQuery` | `conn.table('schema.table')` | Same target, no imported copy |\n| `table X` whose partition is a DAX expression | Computed source, or push upstream | A calculated table |\n| `/// description text` | `#(doc)` tag | Triple-slash doc comment, directly above the object |\n| `isHidden: true` (or bare `isHidden`) | **not a direct mapping** | Classify the reason first; see `rls-roles.md`. Do NOT map it to `internal:` mechanically: Power BI hides surrogate keys by default and `internal` fields are unreachable through join paths. |\n| `role R` + `metadataPermission` / `columnPermission` | `private:` / `internal:` | Object-level security, a real permission; see `rls-roles.md` |\n| `displayFolder:` | nothing | Authoring convenience, no Malloy equivalent |\n| `hierarchy` with `level` | nothing structural | Note level order as drill intent |\n| `role R` + `tablePermission` | `#(access_filter)` / `#(authorize)` | Narrow grammar; see `rls-roles.md` |\n| `perspective` | access modifiers, or a separate source | A curated subset of the model |\n| `LocalDateTable_<guid>`, `DateTableTemplate_<guid>` | **skip** | Auto-generated, one per date column |\n\n## Relationships\n\nA Power BI relationship is declared once at model level, not on either table. `fromColumn` is the **many** side, `toColumn` is the **one** side.\n\n**TMDL writes only non-default properties.** A typical relationship is three lines - `relationship <guid>`, `fromColumn:`, `toColumn:` - and the absence of the properties below is meaningful, not missing data. Read absent as **many-to-one, single-direction, active**. The relationship name is a GUID and carries no information. The same shorthand applies to booleans generally: `isHidden` alone implies `true`, so a grep for `isHidden: true` misses every one written that way.\n\n| Power BI | Malloy | Notes |\n|--------|--------|-------|\n| `fromCardinality: many` / `toCardinality: one` | `join_one:` on the fact source | The common case |\n| `fromCardinality: one` / `toCardinality: one` | `join_one:` | Direct mapping |\n| `fromCardinality: many` / `toCardinality: many` | **flag** | No direct equivalent; changes results |\n| `crossFilteringBehavior: oneDirection` (default) | ordinary `join_one:` | Filter flows from the one side to the many side |\n| `crossFilteringBehavior: bothDirections` | **flag** | No Malloy equivalent; ask what it was for |\n| `isActive: false` | **flag** | Exists for `USERELATIONSHIP`; find the measures that use it |\n| `relyOnReferentialIntegrity: true` | nothing | A query-plan hint |\n\n## Column Types\n\n| Power BI `dataType` | Malloy | Notes |\n|--------|--------|-------|\n| `string` | `string` | Direct mapping |\n| `int64` | `number` | Direct mapping |\n| `double` | `number` | Direct mapping |\n| `decimal` | `number` | Fixed-point in Power BI; check precision if money |\n| `boolean` | `boolean` | Direct mapping |\n| `dateTime` | `date` or `timestamp` | Pick by whether a time component is used |\n| `binary` | **skip** | Not analyzable |\n\n## Aggregations\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `SUM(T[c])` | `sum(c)` | Direct mapping |\n| `AVERAGE(T[c])` | `avg(c)` | Direct mapping |\n| `MIN` / `MAX` | `min()` / `max()` | Direct mapping |\n| `SUMX(T, <row expression>)` | `sum(<expression>)` | **The common case, and it is a direct mapping.** `SUMX(Sales, Sales[Qty] * Sales[Price])` is `sum(qty * price)`. No context transition is involved. |\n| `AVERAGEX` / `MINX` / `MAXX` / `COUNTX` over a row expression | `avg()` / `min()` / `max()` / `count()` | Same: an iterator over a row-level expression is an ordinary aggregate |\n| `SUMX(T, [Some Measure])` | **flag** | A measure reference inside the iterator *is* context transition; see `translate-measures.md` |\n| `RELATED(Other[c])` | `other.c` | Follows a relationship; becomes a join path |\n| `RELATEDTABLE(Other)` | the joined source | Usually an aggregate over a `join_many:` |\n| `SELECTEDVALUE(T[c])` | **flag** | Depends on the query's grouping; no measure-level equivalent |\n| `VAR x = ... RETURN ...` | inline, or a `dimension:` | A local binding; translate the body |\n| `COUNTROWS(T)` | `count()` | Direct mapping |\n| `DISTINCTCOUNT(T[c])` | `count(c)` | **`count(field)` is already the distinct count in Malloy.** `count(distinct c)` is a hard error, despite a message that reads like a deprecation warning: ``count(distinct expression)` deprecated, use `count(expression)` instead.` |\n| `COUNT(T[c])` | `count() { where: c is not null }` | **Not `count(c)`.** DAX `COUNT` counts non-blank rows; Malloy's `count(c)` counts *distinct* values. They differ on any column with repeats, and the wrong one compiles. |\n| `DIVIDE(a, b)` | `a / nullif(b, 0)` | `DIVIDE` returns blank on divide-by-zero |\n| `DIVIDE(a, b, alt)` | `(a / nullif(b, 0)) ?? alt` | The third argument is the divide-by-zero result |\n| `COALESCE(a, b)` | `a ?? b` | Direct mapping |\n| `IF(cond, a, b)` | `pick a when cond else b` | Direct syntax translation |\n| `SWITCH(TRUE(), c1, v1, ...)` | chained `pick ... when` | Direct syntax translation |\n| `BLANK()` | `null` | Not identical in arithmetic; see below |\n\n**`BLANK()` is not `null`.** In DAX, `BLANK() + 1` is `1`; in SQL and Malloy, `null + 1` is `null`. A measure that leans on blank-as-zero arithmetic changes behavior on translation. Check any measure that adds or subtracts other measures.\n\n## Filter Context\n\n**This section is the one that produces wrong numbers.** Read `translate-measures.md` before translating anything that uses `CALCULATE`.\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `CALCULATE(expr, T[c] = \"x\")` | `measure { where: c = 'x' }` **only when safe** | DAX **overwrites** the filter on `T[c]`; Malloy **intersects**. Different answers. |\n| `CALCULATE(expr, KEEPFILTERS(T[c] = \"x\"))` | `measure { where: c = 'x' }` | `KEEPFILTERS` makes DAX intersect, which is what Malloy already does. Safe. |\n| `CALCULATE(expr, ALLSELECTED())` | `all(expr)` | **Exact match for this top-level shape.** `all()` removes *grouping* and keeps the query's `where:` - which is `ALLSELECTED` semantics. Inside a nested iterator `ALLSELECTED` has shadow-filter-context behavior `all()` does not model. `cookbook-filter-context.md#fc3` |\n| `CALCULATE(expr, ALLSELECTED(T[c]))` | `exclude(expr, c)` | **Not `all()`.** A column argument restores only *that* column's visual filter, so in a category > subcategory matrix `ALLSELECTED(Product[Subcategory])` is the category total. |\n| `CALCULATE(expr, ALL(T))` | `all(expr)` **only when grouping is the sole filter** | **Not equivalent in general** - and `ALL`, not `ALLSELECTED`, is the divergent one. DAX `ALL` removes the *filters* too, but only on `T`'s expanded table: `ALL(FactTable)` clears everything reachable and diverges maximally, `ALL(SomeDimension)` leaves filters on other tables standing. `cookbook-filter-context.md#fc2` |\n| `CALCULATE(expr, ALL(T[c]))` | `exclude(expr, c)` **with the same caveat** | Same divergence, one dimension. `cookbook-filter-context.md#fc5` |\n| `CALCULATE(expr, REMOVEFILTERS(T[c]))` | `exclude(expr, c)` **with the same caveat** | `REMOVEFILTERS` is the clearer spelling of `ALL` as a modifier |\n| `CALCULATE(expr, ALLEXCEPT(T, T[keep]))` | `all(expr, keep)` | **Keeps** the listed columns and removes the rest, so it is `all(..., keep)`, not `exclude(..., keep)`. `cookbook-filter-context.md#fc4` |\n| `expr / CALCULATE(expr, ALL(T))` | percent-of-total with `all()` | Matches only under the caveat below. `cookbook-filter-context.md#fc2` |\n| `USERELATIONSHIP(...)` | a second join path | Not a gap: name every role instead of switching between them. `cookbook-structure.md#s1` |\n| `CROSSFILTER(..., BOTH)` | **flag** | Sets filter direction inside one measure, so it leaves no trace in `relationships.tmdl`. `cookbook-structure.md#s3` |\n| `FILTER(T, cond)` as a `CALCULATE` argument | depends | A table filter, not a column filter; see `translate-measures.md` |\n| `EARLIER` / `EARLIEST` | **no recipe** (`NR`) | Row-context construct with no equivalent; ask what the number means and rewrite the intent |\n| `RANKX`, `TOPN` | `calculate: rank()` in a query | Not a measure, but not a gap either. `rank()` orders by any expression, independently of the query's own ordering. `cookbook-filter-context.md#fc6` |\n| `SUMX(FILTER(T, cond), expr)` | `sum(expr) { where: cond }`, with the `FILTER` caveat | Only a context transition if a measure reference appears inside; see `translate-measures.md` |\n| `GENERATESERIES` parameter table + `SELECTEDVALUE` | `given:` | A disconnected what-if slicer is a runtime value. `cookbook-structure.md#s4` |\n\n**`ALL` removes filters; `all()` removes grouping.** This is the same overwrite-versus-intersect divergence as the `CALCULATE` row above, wearing different clothes, and it is on the *safe*-looking side of the table.\n\nIn a report grouped by category with a filter `Year = 2023`:\n\n| | DAX `CALCULATE([Sales], ALL(Sales))` | Malloy `all(sales)` |\n|---|---|---|\n| Category filter (from the grouping) | removed | removed |\n| Year filter | **also removed** | **still applied** |\n| Result | total across all years | total for 2023 |\n\nThey agree when the grouping is the only filter in play, which is exactly the unfiltered grand total people check first. Treat every `ALL`/`REMOVEFILTERS` translation as divergent and validate it with a filter active on a column other than the grouping.\n\n**Second constraint:** a dimension named in `all()` or `exclude()` must be an output field of the query, so `exclude(expr, c)` written as a source-level `measure:` only works in queries that `group_by: c`.\n\n## Time Intelligence\n\nEvery one of these is a **rewrite, not a transcription**. They depend on a table marked as the model's date table with a contiguous date column, a concept Malloy does not have. Translate the intent - the intent translates, and mostly without a stopgap.\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `TOTALYTD(expr, Date[Date])`, `DATESYTD` | `calculate: sum_cumulative(expr) { partition_by: year, order_by: period }` | A correct YTD, and right on sparse data. **`year_end_date` picks the partition column** - it defaults to `\"12/31\"`, so the three-argument form is calendar-year, not fiscal. `cookbook-time.md#t1` |\n| `TOTALMTD`, `DATESMTD`, `DATESQTD` | the same, partitioned one level finer | `cookbook-time.md#t1` |\n| `SAMEPERIODLASTYEAR(Date[Date])` | a filtered aggregate per side, over named ranges | `cookbook-time.md#t2` |\n| `DATEADD(Date[Date], -1, MONTH)`, `PREVIOUSMONTH`, `PARALLELPERIOD` | `lag()` **where every period is present**; named ranges where not | `lag()` is positional, so it silently compares across a gap. `cookbook-time.md#t3` |\n| `CALENDAR()`, `CALENDARAUTO()` | a generated date source | `cookbook-time.md#t6` |\n| `CLOSINGBALANCEMONTH` and the semi-additive family (`LASTNONBLANK`, `LASTDATE` and the rest, as a `CALCULATE` filter) | **STOPGAP** - a multi-stage pipeline | `last_value` cannot be a measure. `cookbook-time.md#t5` |\n| densifying a sparse period series | **STOPGAP** - a generated spine plus a join | Nothing in Malloy densifies. `cookbook-time.md#t4` |\n\n## Functions With No Mapping Yet\n\nFound in real Microsoft-published TMDL and not covered by any table above. Counts are measures containing the function in `PBIASEngine` (126 measures). Most are report-layer once you look at them, which is the point: route them through `translate-measures.md` step 1 rather than treating each as a translation problem.\n\n| DAX | Measures | What to do |\n|--------|---:|-------|\n| `FIRSTNONBLANK` / `LASTNONBLANK` | 16 | The semi-additive shape when it is a `CALCULATE` filter (`cookbook-time.md#t5`). None of the 16 is one, so none routes to T5 |\n| `LASTDATE` / `FIRSTDATE` / `ENDOFMONTH` / `STARTOFMONTH` (and the `ENDOF*` / `STARTOF*` family) | 0 | Return a date, not a measure, and the semi-additive use, `CALCULATE([m], LASTDATE(...))`, needs the value as at that date. Malloy has no measure-level equivalent (`last_value` and `first_value` are window functions, legal only in `calculate:`), so it is a multi-stage query, over the T4 spine for the calendar reading. `cookbook-time.md#t5`. As a `DATESINPERIOD` anchor, `LASTDATE` is a period bound and does not route to T5 |\n| `INT` | 15 | `floor()`, or drop it where it only exists to coerce a Boolean to 1/0 |\n| `DATEDIFF` | 11 | `days(a to b)`, `months(a to b)`, and so on. **Not `date_diff`** - that is not a Malloy function (`Unknown function 'date_diff'`), and `(a - b)` on timestamps is not the idiom either |\n| `ISFILTERED` | 11 | Report-layer. The query knows what it grouped by; the measure does not need to |\n| `COUNTA` | 9 | `count() { where: c is not null }` - the same trap as DAX `COUNT` |\n| `ALLNOBLANKROW` | 9 | `all()`, plus a null check if the blank row was load-bearing |\n| `HASONEFILTER` / `HASONEVALUE` | 9 / 2 | Report-layer, almost always guarding a label |\n| `ISBLANK` | 8 | `is null`, but check the `BLANK()`-as-zero caveat below |\n| `ISINSCOPE` | 7 | Report-layer. Drop it - see `cookbook-filter-context.md#fc6` |\n| `CALCULATETABLE` | 7 | A filtered table expression; usually collapses into the surrounding query's `where:`. With a Boolean filter argument it overwrites the column's filter like `CALCULATE` does, and routes to `FC1` |\n| `GROUPBY` | 7 | A `group_by:` stage |\n| `VALUES` | 5 | The distinct values of a column; `group_by:` or `count(c)` depending on use |\n| `PERCENTILE.INC` | 5 | No direct equivalent; use the dialect's percentile via `fn!()` |\n| `ADDCOLUMNS` / `SUMMARIZE` | 5 / 5 | A query stage, not a measure |\n| `CONCATENATEX` | 5 | `string_agg(expr, sep)`, native - no `fn!()` needed; usually report-layer |\n| `STDEV.P` | 2 | `stddev()` where the dialect has it |\n| `CROSSFILTER` | 1 | `cookbook-structure.md#s3` |\n\n## Formatting\n\n| Power BI | Malloy | Notes |\n|--------|--------|-------|\n| `formatString: \"\\\\$#,0.00\"` | `# currency` | Map to Malloy render tags |\n| `formatString: \"0.00%\"` | `# percent` | Map to Malloy render tags |\n| `formatString: \"#,0\"` | `# number=\"#,0\"` | Map to Malloy render tags |\n| `dataCategory: WebUrl` | `# url` | Map to Malloy render tags |\n| `dataCategory: Place` / `City` / `Country` | nothing automatic | Geo hint for chart selection |"
+      body: "# Power BI → Malloy Concept Mapping\n\nReference table for translating Power BI (TMDL / DAX) constructs to Malloy. Referenced by multiple reference files.\n\n## Model Objects\n\n| Power BI (TMDL) | Malloy | Notes |\n|--------|--------|-------|\n| `table X` | `source:` | One source per table |\n| `column Y` | column reference, or `dimension:` when renamed or derived | Plain columns need no declaration |\n| `column Y` with `type: calculated` | `dimension:` with the translated expression | DAX evaluated at refresh; see `translate-measures.md` |\n| `measure M = <DAX>` | `measure:` | The translation is the hard part, not the syntax |\n| `partition P = m` with `mode: import` | `conn.table('schema.table')` | The M code names the real source table |\n| `partition P` with `mode: directQuery` | `conn.table('schema.table')` | Same target, no imported copy |\n| `table X` whose partition is a DAX expression | Computed source, or push upstream | A calculated table |\n| `/// description text` | `#(doc)` tag | Triple-slash doc comment, directly above the object |\n| `isHidden: true` (or bare `isHidden`) | **not a direct mapping** | Classify the reason first; see `rls-roles.md`. Do NOT map it to `internal:` mechanically: Power BI hides surrogate keys by default and `internal` fields are unreachable through join paths. |\n| `role R` + `metadataPermission` / `columnPermission` | `private:` / `internal:` | Object-level security, a real permission; see `rls-roles.md` |\n| `displayFolder:` | nothing | Authoring convenience, no Malloy equivalent |\n| `hierarchy` with `level` | nothing structural | Note level order as drill intent |\n| `role R` + `tablePermission` | `#(access_filter)` / `#(authorize)` | Narrow grammar; see `rls-roles.md` |\n| `perspective` | access modifiers, or a separate source | A curated subset of the model |\n| `LocalDateTable_<guid>`, `DateTableTemplate_<guid>` | **skip** | Auto-generated, one per date column |\n\n## Relationships\n\nA Power BI relationship is declared once at model level, not on either table. `fromColumn` is the **many** side, `toColumn` is the **one** side.\n\n**TMDL writes only non-default properties.** A typical relationship is three lines - `relationship <guid>`, `fromColumn:`, `toColumn:` - and the absence of the properties below is meaningful, not missing data. Read absent as **many-to-one, single-direction, active**. The relationship name is a GUID and carries no information. The same shorthand applies to booleans generally: `isHidden` alone implies `true`, so a grep for `isHidden: true` misses every one written that way.\n\n| Power BI | Malloy | Notes |\n|--------|--------|-------|\n| `fromCardinality: many` / `toCardinality: one` | `join_one:` on the fact source | The common case |\n| `fromCardinality: one` / `toCardinality: one` | `join_one:` | Direct mapping |\n| `fromCardinality: many` / `toCardinality: many` | **flag** | No direct equivalent; changes results |\n| `crossFilteringBehavior: oneDirection` (default) | ordinary `join_one:` | Filter flows from the one side to the many side |\n| `crossFilteringBehavior: bothDirections` | **flag** | No Malloy equivalent; ask what it was for |\n| `isActive: false` | **flag** | Exists for `USERELATIONSHIP`; find the measures that use it |\n| `relyOnReferentialIntegrity: true` | nothing | A query-plan hint |\n\n## Column Types\n\n| Power BI `dataType` | Malloy | Notes |\n|--------|--------|-------|\n| `string` | `string` | Direct mapping |\n| `int64` | `number` | Direct mapping |\n| `double` | `number` | Direct mapping |\n| `decimal` | `number` | Fixed-point in Power BI; check precision if money |\n| `boolean` | `boolean` | Direct mapping |\n| `dateTime` | `date` or `timestamp` | Pick by whether a time component is used |\n| `binary` | **skip** | Not analyzable |\n\n## Aggregations\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `SUM(T[c])` | `sum(c)` | Direct mapping |\n| `AVERAGE(T[c])` | `avg(c)` | Direct mapping |\n| `MIN` / `MAX` | `min()` / `max()` | Direct mapping |\n| `SUMX(T, <row expression>)` | `sum(<expression>)` | **The common case, and it is a direct mapping.** `SUMX(Sales, Sales[Qty] * Sales[Price])` is `sum(qty * price)`. No context transition is involved. |\n| `AVERAGEX` / `MINX` / `MAXX` / `COUNTX` over a row expression | `avg()` / `min()` / `max()` / `count()` | Same: an iterator over a row-level expression is an ordinary aggregate |\n| `SUMX(T, [Some Measure])` | **flag** | A measure reference inside the iterator *is* context transition; see `translate-measures.md` |\n| `RELATED(Other[c])` | `other.c` | Follows a relationship; becomes a join path |\n| `RELATEDTABLE(Other)` | the joined source | Usually an aggregate over a `join_many:` |\n| `SELECTEDVALUE(T[c])` | **flag** | Depends on the query's grouping; no measure-level equivalent |\n| `VAR x = ... RETURN ...` | inline, or a `dimension:` | A local binding; translate the body |\n| `COUNTROWS(T)` | `count()` | Direct mapping |\n| `DISTINCTCOUNT(T[c])` | `count(c)` | **`count(field)` is already the distinct count in Malloy.** `count(distinct c)` is a hard error, despite a message that reads like a deprecation warning: ``count(distinct expression)` deprecated, use `count(expression)` instead.` |\n| `COUNT(T[c])` | `count() { where: c is not null }` | **Not `count(c)`.** DAX `COUNT` counts non-blank rows; Malloy's `count(c)` counts *distinct* values. They differ on any column with repeats, and the wrong one compiles. |\n| `DIVIDE(a, b)` | `a / nullif(b, 0)` | `DIVIDE` returns blank on divide-by-zero |\n| `DIVIDE(a, b, alt)` | `(a / nullif(b, 0)) ?? alt` | The third argument is the divide-by-zero result |\n| `COALESCE(a, b)` | `a ?? b` | Direct mapping |\n| `IF(cond, a, b)` | `pick a when cond else b` | Direct syntax translation |\n| `SWITCH(TRUE(), c1, v1, ...)` | chained `pick ... when` | Direct syntax translation |\n| `BLANK()` | `null` | Not identical in arithmetic; see below |\n\n**`BLANK()` is not `null`.** In DAX, `BLANK() + 1` is `1`; in SQL and Malloy, `null + 1` is `null`. A measure that leans on blank-as-zero arithmetic changes behavior on translation. Check any measure that adds or subtracts other measures.\n\n## Filter Context\n\n**This section is the one that produces wrong numbers.** Read `translate-measures.md` before translating anything that uses `CALCULATE`.\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `CALCULATE(expr, T[c] = \"x\")` | `measure { where: c = 'x' }` **only when safe** | DAX **overwrites** the filter on `T[c]`; Malloy **intersects**. Different answers. |\n| `CALCULATE(expr, KEEPFILTERS(T[c] = \"x\"))` | `measure { where: c = 'x' }` | `KEEPFILTERS` makes DAX intersect, which is what Malloy already does. Safe. |\n| `CALCULATE(expr, ALLSELECTED())` | `all(expr)` | **Exact match for this top-level shape.** `all()` removes *grouping* and keeps the query's `where:` - which is `ALLSELECTED` semantics. Inside a nested iterator `ALLSELECTED` has shadow-filter-context behavior `all()` does not model. `cookbook-filter-context.md#fc3` |\n| `CALCULATE(expr, ALLSELECTED(T[c]))` | `exclude(expr, c)` | **Not `all()`.** A column argument restores only *that* column's visual filter, so in a category > subcategory matrix `ALLSELECTED(Product[Subcategory])` is the category total. |\n| `CALCULATE(expr, ALL(T))` | `all(expr)` **only when grouping is the sole filter** | **Not equivalent in general** - and `ALL`, not `ALLSELECTED`, is the divergent one. DAX `ALL` removes the *filters* too, but only on `T`'s expanded table: `ALL(FactTable)` clears everything reachable and diverges maximally, `ALL(SomeDimension)` leaves filters on other tables standing. `cookbook-filter-context.md#fc2` |\n| `CALCULATE(expr, ALL(T[c]))` | `exclude(expr, c)` **with the same caveat** | Same divergence, one dimension. `cookbook-filter-context.md#fc5` |\n| `CALCULATE(expr, REMOVEFILTERS(T[c]))` | `exclude(expr, c)` **with the same caveat** | `REMOVEFILTERS` is the clearer spelling of `ALL` as a modifier |\n| `CALCULATE(expr, ALLEXCEPT(T, T[keep]))` | `all(expr, keep)` | **Keeps** the listed columns and removes the rest, so it is `all(..., keep)`, not `exclude(..., keep)`. `cookbook-filter-context.md#fc4` |\n| `expr / CALCULATE(expr, ALL(T))` | percent-of-total with `all()` | Matches only under the caveat below. `cookbook-filter-context.md#fc2` |\n| `USERELATIONSHIP(...)` | a second join path | Not a gap: name every role instead of switching between them. `cookbook-structure.md#s1` |\n| `CROSSFILTER(..., BOTH)` | **flag** | Sets filter direction inside one measure, so it leaves no trace in `relationships.tmdl`. `cookbook-structure.md#s3` |\n| `FILTER(T, cond)` as a `CALCULATE` argument | depends | A table filter, not a column filter; see `translate-measures.md` |\n| `EARLIER` / `EARLIEST` | **no recipe** (`NR`) | Row-context construct with no equivalent; ask what the number means and rewrite the intent |\n| `RANKX`, `TOPN` | `calculate: rank()` in a query | Not a measure, but not a gap either. `rank()` orders by any expression, independently of the query's own ordering. `cookbook-filter-context.md#fc6` |\n| `SUMX(FILTER(T, cond), expr)` | `sum(expr) { where: cond }`, with the `FILTER` caveat | Only a context transition if a measure reference appears inside; see `translate-measures.md` |\n| `GENERATESERIES` parameter table + `SELECTEDVALUE` | `given:` | A disconnected what-if slicer is a runtime value. `cookbook-structure.md#s4` |\n\n**`ALL` removes filters; `all()` removes grouping.** This is the same overwrite-versus-intersect divergence as the `CALCULATE` row above, wearing different clothes, and it is on the *safe*-looking side of the table.\n\nIn a report grouped by category with a filter `Year = 2023`:\n\n| | DAX `CALCULATE([Sales], ALL(Sales))` | Malloy `all(sales)` |\n|---|---|---|\n| Category filter (from the grouping) | removed | removed |\n| Year filter | **also removed** | **still applied** |\n| Result | total across all years | total for 2023 |\n\nThey agree when the grouping is the only filter in play, which is exactly the unfiltered grand total people check first. Treat every `ALL`/`REMOVEFILTERS` translation as divergent and validate it with a filter active on a column other than the grouping.\n\n**Second constraint:** a dimension named in `all()` or `exclude()` must be an output field of the query, so `exclude(expr, c)` written as a source-level `measure:` only works in queries that `group_by: c`.\n\n## Time Intelligence\n\nEvery one of these is a **rewrite, not a transcription**. They depend on a table marked as the model's date table with a contiguous date column, a concept Malloy does not have. Translate the intent - the intent translates, and mostly without a stopgap.\n\n| DAX | Malloy | Notes |\n|--------|--------|-------|\n| `TOTALYTD(expr, Date[Date])`, `DATESYTD` | `calculate: sum_cumulative(expr) { partition_by: year, order_by: period }` | A correct YTD, and right on sparse data. **`year_end_date` picks the partition column** - it defaults to `\"12/31\"`, so the three-argument form is calendar-year, not fiscal. `cookbook-time.md#t1` |\n| `TOTALMTD`, `DATESMTD`, `DATESQTD` | the same, partitioned one level finer | `cookbook-time.md#t1` |\n| `SAMEPERIODLASTYEAR(Date[Date])` | a filtered aggregate per side, over named ranges | `cookbook-time.md#t2` |\n| `DATEADD(Date[Date], -1, MONTH)`, `PREVIOUSMONTH`, `PARALLELPERIOD` | `lag()` **where every period is present**; named ranges where not | `lag()` is positional, so it silently compares across a gap. `cookbook-time.md#t3` |\n| `CALENDAR()`, `CALENDARAUTO()` | a generated date source | `cookbook-time.md#t6` |\n| `CLOSINGBALANCEMONTH` and the semi-additive family (`LASTNONBLANK`, `LASTDATE` and the rest, as a `CALCULATE` filter) | **STOPGAP** - a multi-stage pipeline | `last_value` cannot be a measure. `cookbook-time.md#t5` |\n| densifying a sparse period series | **STOPGAP** - a generated spine plus a join | Nothing in Malloy densifies. `cookbook-time.md#t4` |\n\n## Functions With No Mapping Yet\n\nFound in real Microsoft-published TMDL and not covered by any table above. Counts are measures containing the function in `PBIASEngine` (126 measures). Most are report-layer once you look at them, which is the point: route them through `translate-measures.md` step 1 rather than treating each as a translation problem.\n\n| DAX | Measures | What to do |\n|--------|---:|-------|\n| `FIRSTNONBLANK` / `LASTNONBLANK` | 16 | The semi-additive shape when it is a `CALCULATE` filter (`cookbook-time.md#t5`). None of the 16 is one, so none routes to T5 |\n| `LASTDATE` / `FIRSTDATE` / `ENDOFMONTH` / `STARTOFMONTH` (and the `ENDOF*` / `STARTOF*` family) | 0 | Return a date, not a measure, and the semi-additive use, `CALCULATE([m], LASTDATE(...))`, needs the value as at that date. Malloy has no measure-level equivalent (`last_value` and `first_value` are window functions, legal only in `calculate:`), so it is a multi-stage query, over the T4 spine for the calendar reading. `cookbook-time.md#t5`. As a `DATESINPERIOD` anchor, `LASTDATE` is a period bound and does not route to T5 |\n| `INT` | 15 | `floor()`, or drop it where it only exists to coerce a Boolean to 1/0 |\n| `DATEDIFF` | 11 | `days(a to b)`, `months(a to b)`, and so on. **Not `date_diff`** - that is not a Malloy function (`Unknown function 'date_diff'`), and `(a - b)` on timestamps is not the idiom either |\n| `ISFILTERED` | 11 | Report-layer. The query knows what it grouped by; the measure does not need to |\n| `COUNTA` | 9 | `count() { where: c is not null }` - the same trap as DAX `COUNT` |\n| `ALLNOBLANKROW` | 9 | `all()`, plus a null check if the blank row was load-bearing |\n| `HASONEFILTER` / `HASONEVALUE` | 9 / 2 | Report-layer, almost always guarding a label |\n| `ISBLANK` | 8 | `is null`, but check the `BLANK()`-as-zero caveat below |\n| `ISINSCOPE` | 7 | Report-layer. Drop it - see `cookbook-filter-context.md#fc6` |\n| `CALCULATETABLE` | 7 | A filtered table expression; usually collapses into the surrounding query's `where:`. With a Boolean filter argument it overwrites the column's filter like `CALCULATE` does, and routes to `FC1` |\n| `GROUPBY` | 7 | A `group_by:` stage |\n| `VALUES` | 5 | The distinct values of a column; `group_by:` or `count(c)` depending on use |\n| `PERCENTILE.INC` | 5 | No measure equivalent. A raw-SQL percentile through `fn!()` does not compile as a measure (`skill:malloy-gotchas-modeling`). Compute it as a query with the **Tier boundaries** pattern in `skill:malloy-discover`, or list it as a gap. That pattern is nearest-rank, and `PERCENTILE.INC` interpolates, so the two can differ when the rank falls between two values |\n| `ADDCOLUMNS` / `SUMMARIZE` | 5 / 5 | A query stage, not a measure |\n| `CONCATENATEX` | 5 | `string_agg(expr, sep)`, native - no `fn!()` needed; usually report-layer |\n| `STDEV.P` | 2 | `stddev()` where the dialect has it |\n| `CROSSFILTER` | 1 | `cookbook-structure.md#s3` |\n\n## Formatting\n\n| Power BI | Malloy | Notes |\n|--------|--------|-------|\n| `formatString: \"\\\\$#,0.00\"` | `# currency` | Map to Malloy render tags |\n| `formatString: \"0.00%\"` | `# percent` | Map to Malloy render tags |\n| `formatString: \"#,0\"` | `# number=\"#,0\"` | Map to Malloy render tags |\n| `dataCategory: WebUrl` | `# url` | Map to Malloy render tags |\n| `dataCategory: Place` / `City` / `Country` | nothing automatic | Geo hint for chart selection |"
     },
     {
       name: "malloy-powerbi-review/cookbook-filter-context",