@malloy-publisher/server 0.0.233 → 0.0.235

package/dist/server.mjs

@@ -261151,7 +261151,7 @@ var import_cors = __toESM(require_lib7(), 1);
 var import_express = __toESM(require_express(), 1);
 var import_http_proxy_middleware = __toESM(require_dist4(), 1);
 import * as http2 from "http";
-import * as path12 from "path";
+import * as path13 from "path";
 import { fileURLToPath as fileURLToPath7 } from "url";
 
 // src/controller/compile.controller.ts
@@ -261382,6 +261382,15 @@ var getPersistStorageMode = () => {
   throw new Error(`PERSIST_STORAGE_MODE must be one of ${PERSIST_STORAGE_MODES.join(" | ")} (got ${JSON.stringify(raw)})`);
 };
 var getPersistCollisionEnforce = () => parseBoolEnv("PERSIST_COLLISION_ENFORCE") ?? false;
+var getQueryMetadataMode = () => {
+  const raw = process.env.PUBLISHER_QUERY_METADATA;
+  if (raw === undefined || raw.trim() === "")
+    return "off";
+  const value = raw.trim().toLowerCase();
+  if (value === "on" || value === "off")
+    return value;
+  throw new Error(`PUBLISHER_QUERY_METADATA must be on | off (got ${JSON.stringify(raw)})`);
+};
 function substituteEnvVars(value) {
   const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
   return value.replace(envVarPattern, (_match, varName) => {
@@ -265176,6 +265185,7 @@ function safeJoinUnderRoot(root, ...segments) {
 }
 
 // src/service/connection_config.ts
+init_logger();
 import { createPrivateKey } from "crypto";
 import { existsSync } from "fs";
 import path3 from "path";
@@ -265307,6 +265317,204 @@ function openSshProxy(ssh, target) {
   });
 }
 
+// src/query_metadata_metrics.ts
+var resetHooks = [];
+function lazyCounter(name, description) {
+  let instrument2 = null;
+  resetHooks.push(() => instrument2 = null);
+  return () => instrument2 ??= publisherMeter().createCounter(name, { description });
+}
+var appliedCounter = lazyCounter("publisher_query_metadata_applied_total", "Queries issued with per-query metadata attached. Label: class ('interactive'|'materialize'|'index'|'ops'|'unknown').");
+var droppedCounter = lazyCounter("publisher_query_metadata_properties_dropped_total", "Metadata properties dropped while resolving a bag. Label: reason ('invalid_name'|'invalid_value'|'property_cap'|'serialized_cap').");
+function recordQueryMetadataApplied(queryClass) {
+  appliedCounter().add(1, { class: queryClass });
+}
+function recordQueryMetadataDropped(reason) {
+  droppedCounter().add(1, { reason });
+}
+
+// src/service/query_metadata.ts
+var QUERY_CLASSES = [
+  "interactive",
+  "materialize",
+  "index",
+  "ops"
+];
+var MAX_PROPERTY_NAME_LENGTH = 128;
+var MAX_PROPERTY_VALUE_LENGTH = 256;
+var MAX_PROPERTIES = 20;
+var PROPERTY_NAME_RE = /^[A-Za-z0-9_]+$/;
+var MAX_SERIALIZED_LENGTH = 2000;
+function isRenderableValue(value) {
+  for (let i = 0;i < value.length; i++) {
+    const code = value.charCodeAt(i);
+    if (code < 32 || code > 126 || code === 34)
+      return false;
+  }
+  return true;
+}
+function queryMetadataViolations(raw) {
+  if (raw === undefined || raw === null)
+    return [];
+  if (typeof raw !== "object" || Array.isArray(raw)) {
+    return ["queryMetadata must be an object of string properties"];
+  }
+  const problems = [];
+  const entries = Object.entries(raw);
+  if (entries.length > MAX_PROPERTIES) {
+    problems.push(`queryMetadata declares ${entries.length} properties; at most ${MAX_PROPERTIES} are allowed`);
+  }
+  for (const [name, value] of entries) {
+    if (!PROPERTY_NAME_RE.test(name)) {
+      problems.push(`queryMetadata property name '${name}' must be ASCII alphanumerics and underscore`);
+    } else if (name.length > MAX_PROPERTY_NAME_LENGTH) {
+      problems.push(`queryMetadata property name '${name}' exceeds ${MAX_PROPERTY_NAME_LENGTH} characters`);
+    }
+    if (typeof value !== "string") {
+      problems.push(`queryMetadata property '${name}' must be a string (got ${typeof value})`);
+      continue;
+    }
+    if (!isRenderableValue(value)) {
+      problems.push(`queryMetadata property '${name}' has a value outside printable ASCII, or containing '"'`);
+    }
+    if (value.length > MAX_PROPERTY_VALUE_LENGTH) {
+      problems.push(`queryMetadata property '${name}' value exceeds ${MAX_PROPERTY_VALUE_LENGTH} characters`);
+    }
+  }
+  return problems;
+}
+function queryMetadataAdvisoryWarnings(meta) {
+  const warnings = [];
+  for (const name of Object.keys(meta)) {
+    if (!/^[A-Za-z]/.test(name)) {
+      warnings.push(`queryMetadata property '${name}' does not start with a letter, so BigQuery drops it (other backends keep it); rename it to start with a letter`);
+    }
+  }
+  return warnings;
+}
+function queryMetadataBudgetWarning(declared) {
+  const authorBudget = MAX_PROPERTIES - RESERVED_CONTEXT_PROPERTIES;
+  if (declared <= authorBudget)
+    return;
+  return `queryMetadata declares ${declared} properties; the server adds up to ` + `${RESERVED_CONTEXT_PROPERTIES} of its own to every statement, so a bag over ` + `${authorBudget} loses its least specific properties at query time`;
+}
+function mintCorrelationId() {
+  return crypto.randomUUID();
+}
+var CONTEXT_SHED_ORDER = [
+  "version",
+  "model",
+  "source",
+  "trigger",
+  "environment",
+  "package",
+  "run_id",
+  "query_id",
+  "class"
+];
+var RESERVED_CONTEXT_PROPERTIES = CONTEXT_SHED_ORDER.length;
+function queryContextProperties(context) {
+  const out = {};
+  const put = (name, value) => {
+    if (value !== undefined && value !== "")
+      out[name] = value;
+  };
+  put("class", context.queryClass);
+  put("environment", context.environment);
+  put("package", context.package);
+  put("version", context.version);
+  put("model", context.model);
+  put("source", context.source);
+  put("trigger", context.trigger);
+  put("run_id", context.runId);
+  put("query_id", context.correlationId);
+  return out;
+}
+function sanitizeValue(value) {
+  let out = "";
+  for (let i = 0;i < value.length && out.length < MAX_PROPERTY_VALUE_LENGTH; i++) {
+    const code = value.charCodeAt(i);
+    out += code < 32 || code > 126 || code === 34 ? "_" : value[i];
+  }
+  return out;
+}
+function mergeQueryMetadata(layers) {
+  if (getQueryMetadataMode() === "off")
+    return { drops: [] };
+  const drops = [];
+  const contextProperties = queryContextProperties(layers.context ?? {});
+  const merged = Object.create(null);
+  const winningLayer = new Map;
+  const orderedLayers = [
+    layers.connection,
+    layers.model,
+    layers.request,
+    layers.enforced
+  ];
+  orderedLayers.forEach((layer, index) => {
+    for (const [name, value] of Object.entries(layer ?? {})) {
+      if (!PROPERTY_NAME_RE.test(name) || name.length > MAX_PROPERTY_NAME_LENGTH) {
+        drops.push({ name, reason: "invalid_name" });
+        continue;
+      }
+      if (typeof value !== "string") {
+        drops.push({ name, reason: "invalid_value" });
+        continue;
+      }
+      winningLayer.set(name, index);
+      merged[name] = sanitizeValue(value);
+    }
+  });
+  for (const [name, value] of Object.entries(contextProperties)) {
+    merged[name] = sanitizeValue(value);
+  }
+  const shedOrder = [...winningLayer.entries()].filter(([name]) => !Object.prototype.hasOwnProperty.call(contextProperties, name)).sort((a, b) => a[1] - b[1]).map(([name]) => name);
+  const shed = (reason) => {
+    const name = shedOrder.shift();
+    delete merged[name];
+    drops.push({ name, reason });
+  };
+  while (Object.keys(merged).length > MAX_PROPERTIES && shedOrder.length > 0) {
+    shed("property_cap");
+  }
+  while (JSON.stringify(merged).length > MAX_SERIALIZED_LENGTH && shedOrder.length > 0) {
+    shed("serialized_cap");
+  }
+  const contextShedOrder = CONTEXT_SHED_ORDER.filter((name) => (name in contextProperties));
+  while ((Object.keys(merged).length > MAX_PROPERTIES || JSON.stringify(merged).length > MAX_SERIALIZED_LENGTH) && contextShedOrder.length > 0) {
+    const name = contextShedOrder.shift();
+    const overCount = Object.keys(merged).length > MAX_PROPERTIES;
+    delete merged[name];
+    drops.push({
+      name,
+      reason: overCount ? "property_cap" : "serialized_cap"
+    });
+  }
+  for (const drop of drops)
+    recordQueryMetadataDropped(drop.reason);
+  if (Object.keys(merged).length === 0)
+    return { drops };
+  recordQueryMetadataApplied(layers.context?.queryClass ?? "unknown");
+  return { metadata: merged, drops };
+}
+function parseSuppliedQueryMetadata(raw) {
+  if (raw === undefined || raw === null)
+    return;
+  const problems = queryMetadataViolations(raw);
+  if (problems.length > 0)
+    throw new Error(problems.join("; "));
+  const meta = raw;
+  return Object.keys(meta).length > 0 ? meta : undefined;
+}
+function parseQueryClass(raw) {
+  if (raw === undefined || raw === null)
+    return;
+  if (typeof raw === "string" && QUERY_CLASSES.includes(raw)) {
+    return raw;
+  }
+  throw new Error(`queryClass must be one of ${QUERY_CLASSES.join(" | ")} (got ${JSON.stringify(raw)})`);
+}
+
 // src/service/connection_config.ts
 var PROXIED_SSLMODES = [
   "disable",
@@ -265481,6 +265689,33 @@ function buildDuckdbEntry(name, environmentPath, databaseFilename = `${name}.duc
     databasePath: path3.join(environmentPath, databaseFilename)
   };
 }
+function warnOnConnectionQueryMetadata(connection) {
+  let declared = 0;
+  for (const field of ["queryMetadata", "queryMetadataEnforced"]) {
+    const metadata = connection[field];
+    if (!metadata)
+      continue;
+    declared += Object.keys(metadata).length;
+    const problems = [
+      ...queryMetadataViolations(metadata),
+      ...queryMetadataAdvisoryWarnings(metadata)
+    ];
+    for (const problem of problems) {
+      logger.warn("Connection query metadata will not apply as declared", {
+        connectionName: connection.name,
+        field,
+        problem
+      });
+    }
+  }
+  const overBudget = queryMetadataBudgetWarning(declared);
+  if (overBudget) {
+    logger.warn("Connection query metadata will not apply as declared", {
+      connectionName: connection.name,
+      problem: overBudget
+    });
+  }
+}
 function validateConnectionShape(connection) {
   if (connection.proxy) {
     if (connection.proxy.type !== "ssh") {
@@ -265648,6 +265883,7 @@ function assembleEnvironmentConnections(connections = [], environmentPath = "")
     processedConnections.add(connection.name);
     validateDuckdbApiSurface(connection);
     validateConnectionShape(connection);
+    warnOnConnectionQueryMetadata(connection);
     const apiConnection = cloneApiConnection(connection);
     apiConnection.attributes = getStaticConnectionAttributes(connection.type);
     const attachedDatabases = connection.duckdbConnection?.attachedDatabases ?? [];
@@ -268261,6 +268497,12 @@ function validateAdminAuthoredConnection(connectionName, connectionConfig) {
   } catch (error) {
     throw new BadRequestError(error.message);
   }
+  for (const field of ["queryMetadata", "queryMetadataEnforced"]) {
+    const violations = queryMetadataViolations(connectionConfig[field]);
+    if (violations.length > 0) {
+      throw new BadRequestError(`Connection "${connectionName}" ${field} is invalid: ${violations.join("; ")}`);
+    }
+  }
 }
 
 class ConnectionController {
@@ -268280,6 +268522,22 @@ class ConnectionController {
     }
     return environment.getApiConnection(connectionName);
   }
+  async connectionQueryMetadata(environmentName, connectionName) {
+    try {
+      const environment = await this.environmentStore.getEnvironment(environmentName, false);
+      const connection = this.getApiConnectionForLookup(environment, connectionName);
+      return {
+        default: connection.queryMetadata ?? null,
+        enforced: connection.queryMetadataEnforced ?? null
+      };
+    } catch (error) {
+      logger.debug("No query-metadata layers for connection", {
+        connectionName,
+        error
+      });
+      return { default: null, enforced: null };
+    }
+  }
   async getMalloyConnection(environmentName, connectionName, packageName) {
     const environment = await this.environmentStore.getEnvironment(environmentName, false);
     if (connectionName === "duckdb") {
@@ -268410,7 +268668,7 @@ class ConnectionController {
     }
     return this.fetchTable(malloyConnection, tableKey, tablePath);
   }
-  async getConnectionQueryData(environmentName, connectionName, sqlStatement, options, packageName) {
+  async getConnectionQueryData(environmentName, connectionName, sqlStatement, options, packageName, metadata) {
     if (typeof sqlStatement !== "string") {
       throw new BadRequestError("sqlStatement must be a string");
     }
@@ -268437,6 +268695,29 @@ class ConnectionController {
       logger.info("Clearing unsupported abortSignal");
       runSQLOptions.abortSignal = undefined;
     }
+    const suppliedMetadata = metadata?.queryMetadata ?? runSQLOptions.queryMetadata;
+    let requestMetadata;
+    let queryClass;
+    try {
+      requestMetadata = parseSuppliedQueryMetadata(suppliedMetadata);
+      queryClass = parseQueryClass(metadata?.queryClass);
+    } catch (error) {
+      throw new BadRequestError(error.message);
+    }
+    const connectionLayers = await this.connectionQueryMetadata(environmentName, connectionName);
+    const resolvedMetadata = mergeQueryMetadata({
+      connection: connectionLayers.default,
+      enforced: connectionLayers.enforced,
+      request: requestMetadata,
+      context: {
+        queryClass: queryClass ?? "ops",
+        environment: environmentName,
+        package: packageName,
+        correlationId: mintCorrelationId()
+      }
+    });
+    runSQLOptions.queryMetadata = resolvedMetadata.metadata;
+    const queryCorrelationId = resolvedMetadata.metadata?.query_id ?? null;
     const maxRows = getMaxQueryRows();
     const maxBytes = getMaxResponseBytes();
     if (maxRows > 0) {
@@ -268456,7 +268737,7 @@ class ConnectionController {
           throw new ConnectionError(error.message);
         }
       }, getQueryTimeoutMs());
-      return { data: JSON.stringify(streamed) };
+      return { data: JSON.stringify(streamed), queryCorrelationId };
     }
     const result = await runWithQueryTimeout(async (signal) => {
       const optionsWithSignal = {
@@ -268473,7 +268754,7 @@ class ConnectionController {
       recordQueryCapExceeded("rows", "connection_sql");
       throw new PayloadTooLargeError(`Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`);
     }
-    return { data: JSON.stringify(result) };
+    return { data: JSON.stringify(result), queryCorrelationId };
   }
   async getConnectionTemporaryTable(environmentName, connectionName, sqlStatement, packageName) {
     if (typeof sqlStatement !== "string") {
@@ -268573,6 +268854,7 @@ class DatabaseController {
 
 // src/controller/model.controller.ts
 init_errors();
+init_logger();
 class ModelController {
   environmentStore;
   constructor(environmentStore) {
@@ -268630,7 +268912,21 @@ class ModelController {
     if (model.getType() === "model") {
       throw new ModelNotFoundError(`${notebookPath} is a model`);
     }
-    return runWithQueryTimeout((abortSignal) => model.executeNotebookCell(cellIndex, filterParams, bypassFilters, givens, abortSignal), getQueryTimeoutMs());
+    return runWithQueryTimeout((abortSignal) => model.executeNotebookCell(cellIndex, filterParams, bypassFilters, givens, abortSignal, {
+      environment: environmentName,
+      connectionMetadata: (connectionName) => {
+        try {
+          const connection = environment.getApiConnection(connectionName);
+          return {
+            default: connection.queryMetadata,
+            enforced: connection.queryMetadataEnforced
+          };
+        } catch (error) {
+          logger.debug("No query-metadata layers for connection", { connectionName, error });
+          return null;
+        }
+      }
+    }), getQueryTimeoutMs());
   }
 }
 
@@ -268762,12 +269058,21 @@ function bigIntReplacer(_key, value) {
 }
 
 // src/controller/query.controller.ts
+init_logger();
 class QueryController {
   environmentStore;
   constructor(environmentStore) {
     this.environmentStore = environmentStore;
   }
-  async getQuery(environmentName, packageName, modelPath, sourceName, queryName, query, compactJson = false, filterParams, bypassFilters, givens) {
+  async getQuery(environmentName, packageName, modelPath, sourceName, queryName, query, compactJson = false, filterParams, bypassFilters, givens, metadata) {
+    let requestMetadata;
+    let queryClass;
+    try {
+      requestMetadata = parseSuppliedQueryMetadata(metadata?.queryMetadata);
+      queryClass = parseQueryClass(metadata?.queryClass);
+    } catch (error) {
+      throw new BadRequestError(error.message);
+    }
     const environment = await this.environmentStore.getEnvironment(environmentName, false);
     environment.assertCanAdmitQuery();
     const p = await environment.getPackage(packageName, false);
@@ -268775,14 +269080,39 @@ class QueryController {
     if (!model) {
       throw new ModelNotFoundError(`${modelPath} does not exist`);
     } else {
-      const { result, compactResult, rowLimit, rowLimitSource } = await runWithQueryTimeout((abortSignal) => model.getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal), getQueryTimeoutMs());
+      const {
+        result,
+        compactResult,
+        rowLimit,
+        rowLimitSource,
+        queryCorrelationId
+      } = await runWithQueryTimeout((abortSignal) => model.getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal, {
+        request: requestMetadata,
+        queryClass,
+        environment: environmentName,
+        version: metadata?.versionId,
+        correlationId: mintCorrelationId(),
+        connectionMetadata: (connectionName) => {
+          try {
+            const connection = environment.getApiConnection(connectionName);
+            return {
+              default: connection.queryMetadata,
+              enforced: connection.queryMetadataEnforced
+            };
+          } catch (error) {
+            logger.debug("No query-metadata layers for connection", { connectionName, error });
+            return null;
+          }
+        }
+      }), getQueryTimeoutMs());
       const renderLogs = import_render_validator.validateRenderTags(result);
       return {
         result: compactJson ? JSON.stringify(compactResult, bigIntReplacer) : JSON.stringify(result),
         resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}/models/${modelPath}/query`,
         renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
         queryRowLimit: rowLimit,
-        queryRowLimitSource: rowLimitSource
+        queryRowLimitSource: rowLimitSource,
+        queryCorrelationId
       };
     }
   }
@@ -276155,37 +276485,37 @@ import { pathToFileURL as pathToFileURL2 } from "url";
 init_logger();
 
 // src/materialization_metrics.ts
-var resetHooks = [];
-function lazyCounter(name, description) {
+var resetHooks2 = [];
+function lazyCounter2(name, description) {
   let instrument2 = null;
-  resetHooks.push(() => instrument2 = null);
+  resetHooks2.push(() => instrument2 = null);
   return () => instrument2 ??= publisherMeter().createCounter(name, { description });
 }
 function lazyHistogram(name, description, unit) {
   let instrument2 = null;
-  resetHooks.push(() => instrument2 = null);
+  resetHooks2.push(() => instrument2 = null);
   return () => instrument2 ??= publisherMeter().createHistogram(name, {
     description,
     unit
   });
 }
-var runCounter = lazyCounter("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
+var runCounter = lazyCounter2("publisher_materialization_runs_total", "Materialization builds completed. Labels: mode ('auto'|'orchestrated'), outcome ('success'|'failed'|'cancelled').");
 var runDuration = lazyHistogram("publisher_materialization_run_duration_ms", "Wall-clock duration of a materialization build. Label: mode ('auto'|'orchestrated').", "ms");
-var sourcesCounter = lazyCounter("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
+var sourcesCounter = lazyCounter2("publisher_materialization_sources_total", "Persist sources processed by a materialization run. Label: outcome ('built'|'reused').");
 var buildPlanComputeDuration = lazyHistogram("publisher_materialization_build_plan_compute_duration_ms", "Wall-clock duration of compiling a package's build plan (Package.buildPlan).", "ms");
-var autoLoadCounter = lazyCounter("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
-var connectionDigestSkipCounter = lazyCounter("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
-var manifestBindCounter = lazyCounter("publisher_materialization_manifest_bind_total", "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').");
-var manifestBindDegradedCounter = lazyCounter("publisher_materialization_manifest_bind_degraded_total", "Manifest entries bound with an UNQUOTED table path because their connection " + "could not be resolved (serve-side bind) or is absent from the build " + "(build-side seed). A misconfiguration that breaks the source on a " + "case-folding engine (Snowflake); alertable.");
+var autoLoadCounter = lazyCounter2("publisher_materialization_auto_load_total", "Auto-run manifest auto-load attempts. Label: outcome ('success'|'failure').");
+var connectionDigestSkipCounter = lazyCounter2("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
+var manifestBindCounter = lazyCounter2("publisher_materialization_manifest_bind_total", "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').");
+var manifestBindDegradedCounter = lazyCounter2("publisher_materialization_manifest_bind_degraded_total", "Manifest entries bound with an UNQUOTED table path because their connection " + "could not be resolved (serve-side bind) or is absent from the build " + "(build-side seed). A misconfiguration that breaks the source on a " + "case-folding engine (Snowflake); alertable.");
 var sourceBuildDuration = lazyHistogram("publisher_materialization_source_build_duration_ms", "Wall-clock duration of building a single persist source.", "ms");
-var dropTablesCounter = lazyCounter("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
-var scheduledFireCounter = lazyCounter("publisher_materialization_scheduled_fires_total", "Standalone-scheduler attempts to fire a package's materialization.schedule. " + "Label: outcome ('fired'|'conflict'|'error').");
-var storageServeRoutingCounter = lazyCounter("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|'runtime_live_fallback').");
-var storageBuildFailureCounter = lazyCounter("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Label: destination (connection name).");
-var eligibilityRefusedCounter = lazyCounter("publisher_materialization_eligibility_refused_total", "storage= materialization-eligibility refusals. Label: reason " + "('free_parameter'|'given'|'authorize'|'not_duckdb_portable'|" + "'public_surface_unknown').");
-var serveShapeTierDropCounter = lazyCounter("publisher_storage_serve_shape_tier_drop_total", "storage serve-shape compile escalations: a refinement tier failed to " + "compile and the riskiest category was dropped. Label: tier (the failed " + "tier index, 0=full).");
-var serveShapeTypeFallbackCounter = lazyCounter("publisher_storage_serve_shape_type_fallback_total", "Captured DuckDB column types mapped to json in the serve shape (type " + "fidelity loss). Label: kind ('array'|'unrecognized').");
-var chainedStorageBuildCounter = lazyCounter("publisher_storage_chained_build_total", "Chained storage= source builds (a source reading a storage-materialized " + "upstream). Label: outcome ('parent_reuse'|'inline_fallback'|" + "'strict_refused'). The parent_reuse share is the headline signal for how " + "far the stack-on-the-parent path gets us vs recompute-from-raw.");
+var dropTablesCounter = lazyCounter2("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
+var scheduledFireCounter = lazyCounter2("publisher_materialization_scheduled_fires_total", "Standalone-scheduler attempts to fire a package's materialization.schedule. " + "Label: outcome ('fired'|'conflict'|'error').");
+var storageServeRoutingCounter = lazyCounter2("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|'runtime_live_fallback').");
+var storageBuildFailureCounter = lazyCounter2("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Label: destination (connection name).");
+var eligibilityRefusedCounter = lazyCounter2("publisher_materialization_eligibility_refused_total", "storage= materialization-eligibility refusals. Label: reason " + "('free_parameter'|'given'|'authorize'|'not_duckdb_portable'|" + "'public_surface_unknown').");
+var serveShapeTierDropCounter = lazyCounter2("publisher_storage_serve_shape_tier_drop_total", "storage serve-shape compile escalations: a refinement tier failed to " + "compile and the riskiest category was dropped. Label: tier (the failed " + "tier index, 0=full).");
+var serveShapeTypeFallbackCounter = lazyCounter2("publisher_storage_serve_shape_type_fallback_total", "Captured DuckDB column types mapped to json in the serve shape (type " + "fidelity loss). Label: kind ('array'|'unrecognized').");
+var chainedStorageBuildCounter = lazyCounter2("publisher_storage_chained_build_total", "Chained storage= source builds (a source reading a storage-materialized " + "upstream). Label: outcome ('parent_reuse'|'inline_fallback'|" + "'strict_refused'). The parent_reuse share is the headline signal for how " + "far the stack-on-the-parent path gets us vs recompute-from-raw.");
 function recordScheduledFire(outcome) {
   scheduledFireCounter().add(1, { outcome });
 }
@@ -278267,7 +278597,7 @@ class Model {
       return { ...b, schema, refinements };
     }).filter((b) => b.schema.length > 0);
   }
-  async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal) {
+  async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal, queryMetadataInput) {
     const startTime = performance.now();
     if (this.compilationError) {
       if (this.compilationError instanceof MalloyError2 || this.compilationError instanceof ModelCompilationError) {
@@ -278375,26 +278705,30 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
     let rowLimitSource = "server_default";
     let executionTime = 0;
     let queryResults;
+    let appliedQueryMetadata;
     const querySurfaceGivens = this.filterGivensToModelSurface(givens);
     const effectiveGivens = serveVirtualMap ? undefined : querySurfaceGivens;
     try {
-      const preparedLimit = (await runnable.getPreparedResult({
+      const preparedResult = await runnable.getPreparedResult({
         givens: effectiveGivens,
         buildManifest: effectiveBuildManifest,
         virtualMap: serveVirtualMap
-      })).resultExplore.limit;
+      });
+      const preparedLimit = preparedResult.resultExplore.limit;
       rowLimitSource = queryRowLimitSource(preparedLimit);
       rowLimit = resolveModelQueryRowLimit(preparedLimit, {
         defaultLimit: getDefaultQueryRowLimit(),
         maxRows
       });
+      appliedQueryMetadata = this.resolveQueryMetadata(queryMetadataInput, preparedResult.connectionName);
       executionTime = performance.now() - startTime;
       queryResults = await runnable.run({
         rowLimit,
         givens: effectiveGivens,
         abortSignal,
         buildManifest: effectiveBuildManifest,
-        virtualMap: serveVirtualMap
+        virtualMap: serveVirtualMap,
+        queryMetadata: appliedQueryMetadata
       });
     } catch (error) {
       const canDegradeToLive = !!serveVirtualMap && !!liveRunnable && !abortSignal?.aborted && !String(error?.code ?? "").startsWith("runtime-given-") && serveShapeBindings.length > 0 && serveShapeBindings.every((b) => b.freshnessFallback === "live");
@@ -278441,20 +278775,23 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       });
       recordStorageServeRouting("runtime_live_fallback");
       try {
-        const livePreparedLimit = (await liveRunnable.getPreparedResult({
+        const livePrepared = await liveRunnable.getPreparedResult({
           givens: querySurfaceGivens,
           buildManifest
-        })).resultExplore.limit;
+        });
+        const livePreparedLimit = livePrepared.resultExplore.limit;
         rowLimitSource = queryRowLimitSource(livePreparedLimit);
         rowLimit = resolveModelQueryRowLimit(livePreparedLimit, {
           defaultLimit: getDefaultQueryRowLimit(),
           maxRows
         });
+        appliedQueryMetadata = this.resolveQueryMetadata(queryMetadataInput, livePrepared.connectionName);
         queryResults = await liveRunnable.run({
           rowLimit,
           givens: querySurfaceGivens,
           abortSignal,
-          buildManifest
+          buildManifest,
+          queryMetadata: appliedQueryMetadata
         });
       } catch (retryError) {
         failQuery(retryError);
@@ -278482,9 +278819,40 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       modelInfo: this.modelInfo,
       dataStyles: this.dataStyles,
       rowLimit,
-      rowLimitSource
+      rowLimitSource,
+      queryCorrelationId: appliedQueryMetadata?.query_id ?? null
     };
   }
+  resolveQueryMetadata(input, connectionName) {
+    let connectionLayers = null;
+    if (connectionName && input?.connectionMetadata) {
+      try {
+        connectionLayers = input.connectionMetadata(connectionName);
+      } catch {
+        connectionLayers = null;
+      }
+    }
+    const resolved = mergeQueryMetadata({
+      connection: connectionLayers?.default,
+      enforced: connectionLayers?.enforced,
+      request: input?.request,
+      context: {
+        queryClass: input?.queryClass ?? "interactive",
+        environment: input?.environment,
+        package: this.packageName,
+        model: this.modelPath,
+        version: input?.version,
+        correlationId: input?.correlationId
+      }
+    });
+    if (resolved.drops.length > 0) {
+      logger.warn("Dropped query-metadata properties for a query", {
+        modelPath: this.modelPath,
+        drops: resolved.drops
+      });
+    }
+    return resolved.metadata;
+  }
   getStandardModel() {
     return {
       type: "source",
@@ -278525,7 +278893,7 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
       notebookCells
     };
   }
-  async executeNotebookCell(cellIndex, filterParams, bypassFilters, givens, abortSignal) {
+  async executeNotebookCell(cellIndex, filterParams, bypassFilters, givens, abortSignal, queryMetadataInput) {
     if (this.compilationError) {
       throw this.compilationError;
     }
@@ -278567,10 +278935,11 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
         const cellMaxBytes = getMaxResponseBytes();
         const buildManifest = this.resolveFreshBuildManifest();
         const cellSurfaceGivens = this.filterGivensToModelSurface(givens);
-        const rowLimit = resolveModelQueryRowLimit((await runnableToExecute.getPreparedResult({
+        const preparedCell = await runnableToExecute.getPreparedResult({
           givens: cellSurfaceGivens,
           buildManifest
-        })).resultExplore.limit, {
+        });
+        const rowLimit = resolveModelQueryRowLimit(preparedCell.resultExplore.limit, {
           defaultLimit: getDefaultQueryRowLimit(),
           maxRows: cellMaxRows
         });
@@ -278578,7 +278947,8 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
           rowLimit,
           givens: cellSurfaceGivens,
           abortSignal,
-          buildManifest
+          buildManifest,
+          queryMetadata: this.resolveQueryMetadata(queryMetadataInput, preparedCell.connectionName)
         });
         const query = (await runnableToExecute.getPreparedQuery())._query;
         queryName = query.as || query.name;
@@ -278917,6 +279287,10 @@ function tagFreshnessLayer(tag) {
   }
   return layer;
 }
+function modelTagLayers(tag) {
+  const envelope = tag && typeof tag.tag === "function" ? tag.tag("materialization") : undefined;
+  return [envelope, tag];
+}
 function packageFreshnessLayer(cfg) {
   if (!cfg)
     return {};
@@ -278944,11 +279318,13 @@ function safeModelTag(source) {
   }
 }
 function resolveFreshness(source, packageMaterialization) {
-  const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
-  const modelLayer = tagFreshnessLayer(safeModelTag(source));
-  const pkgLayer = packageFreshnessLayer(packageMaterialization);
-  const window2 = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
-  const fallback = sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
+  const layers = [
+    tagFreshnessLayer(safeSourceTag(source)),
+    ...modelTagLayers(safeModelTag(source)).map(tagFreshnessLayer),
+    packageFreshnessLayer(packageMaterialization)
+  ];
+  const window2 = layers.map((l) => l.window).find((v) => v !== undefined);
+  const fallback = layers.map((l) => l.fallback).find((v) => v !== undefined);
   if (window2 === undefined && fallback === undefined)
     return null;
   const freshness = {};
@@ -278958,6 +279334,31 @@ function resolveFreshness(source, packageMaterialization) {
     freshness.fallback = fallback;
   return freshness;
 }
+function tagQueryMetadataLayer(tag) {
+  const subtree = tag && typeof tag.tag === "function" ? tag.tag("queryMetadata") : undefined;
+  if (!subtree || typeof subtree.entries !== "function")
+    return {};
+  const layer = {};
+  try {
+    for (const [name, value] of subtree.entries()) {
+      const text = value.text();
+      if (text !== undefined)
+        layer[name] = text;
+    }
+  } catch {
+    return {};
+  }
+  return layer;
+}
+function resolveQueryMetadata(source, packageMaterialization) {
+  const layers = [
+    packageMaterialization?.queryMetadata ?? {},
+    ...modelTagLayers(safeModelTag(source)).map(tagQueryMetadataLayer).reverse(),
+    tagQueryMetadataLayer(safeSourceTag(source))
+  ];
+  const resolved = Object.assign({}, ...layers);
+  return Object.keys(resolved).length > 0 ? resolved : null;
+}
 function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
@@ -279107,6 +279508,7 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       sql: source.getSQL(),
       refresh: annotationFields.refresh ?? null,
       freshness: resolveFreshness(source, packageMaterialization),
+      queryMetadata: resolveQueryMetadata(source, packageMaterialization),
       columns: deriveColumns(source),
       annotationFields,
       modelPath: sourceModelPaths?.[sourceID]
@@ -279138,6 +279540,36 @@ function collectSourceEligibility(sources) {
   return { eligible, refused };
 }
 
+// src/service/materialization_config_validation.ts
+function metadataWarnings(level, metadata, target) {
+  if (!metadata)
+    return [];
+  const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
+  return [
+    ...queryMetadataViolations(metadata),
+    ...queryMetadataAdvisoryWarnings(metadata),
+    ...budget ? [budget] : []
+  ].map((message) => ({
+    message: `${level}: ${message}`,
+    ...target ? { target } : {}
+  }));
+}
+function materializationConfigWarnings(input) {
+  const warnings = (input.manifestWarnings ?? []).map((message) => ({ message }));
+  warnings.push(...metadataWarnings("materialization.queryMetadata", input.packageMaterialization?.queryMetadata));
+  for (const source of input.sources ?? []) {
+    warnings.push(...metadataWarnings(`#@ persist queryMetadata`, source.queryMetadata, source.name));
+  }
+  const seen = new Set;
+  return warnings.filter((warning) => {
+    const key = `${warning.target ?? ""}\x00${warning.message}`;
+    if (seen.has(key))
+      return false;
+    seen.add(key);
+    return true;
+  });
+}
+
 // src/service/cron_evaluator.ts
 var import_cron_parser = __toESM(require_dist14(), 1);
 var UNIX_CRON_NAMES = new Set([
@@ -279205,7 +279637,7 @@ class CronEvaluator {
 // src/service/persist_annotation_validation.ts
 init_errors();
 var PERSIST_LINE_PATTERN = /^\s*#@\s+persist\b/;
-var UNQUOTED_NAME_PATTERN = /\bname\s*=\s*(?!["'])/;
+var UNQUOTED_NAME_PATTERN = /(?<![.\w])name\s*=\s*(?!["'])/;
 function assertPersistNamesQuoted(modelSource, modelPath) {
   const offenders = [];
   for (const rawLine of modelSource.split(`
@@ -279260,6 +279692,7 @@ class Package {
   droppedPersistSources = [];
   sourceEligibility = undefined;
   renderTagWarnings = [];
+  manifestWarnings = [];
   static meter = publisherMeter();
   static packageLoadHistogram = this.meter.createHistogram("malloy_package_load_duration", {
     description: "Time taken to load a Malloy package",
@@ -279428,6 +279861,7 @@ class Package {
     });
     const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
     pkg.renderTagWarnings = renderTagWarnings;
+    pkg.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
     pkg.wireFreshnessResolvers();
     try {
       const buildPlanStart = Date.now();
@@ -279493,7 +279927,12 @@ class Package {
       ...this.renderTagWarnings,
       ...this.storageWarnings(),
       ...this.droppedPersistWarnings(),
-      ...this.persistenceCollisionWarnings().map((message) => ({ message }))
+      ...this.persistenceCollisionWarnings().map((message) => ({ message })),
+      ...materializationConfigWarnings({
+        packageMaterialization: this.packageMetadata.materialization,
+        sources: this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [],
+        manifestWarnings: this.manifestWarnings
+      })
     ];
     if (allWarnings.length > 0) {
       metadata.warnings = allWarnings;
@@ -279640,7 +280079,7 @@ class Package {
     for (const source of sources) {
       const fields = source.annotationFields ?? {};
       if (fields.sharing !== undefined) {
-        warnings.push(`#@ persist source "${source.name}" declares sharing=... which is ` + `no longer supported: scope is a single package-level mode. Set ` + `the root-level "scope": "version" | "package" in ` + `${PACKAGE_MANIFEST_NAME} instead.`);
+        warnings.push(`#@ persist source "${source.name}" declares sharing=... which is ` + `no longer supported: scope is a single package-level mode. Set ` + `"materialization": { "scope": "version" | "package" } in ` + `${PACKAGE_MANIFEST_NAME} instead.`);
       }
       if (fields.schedule !== undefined) {
         warnings.push(`#@ persist source "${source.name}" declares schedule=... which is ` + `no longer supported: a schedule is package-root-only. Declare ` + `"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` + `(requires "scope": "version") instead.`);
@@ -279820,6 +280259,7 @@ class Package {
     this.models = nextModels;
     this.pushStorageServeBindingsToModels();
     this.renderTagWarnings = renderTagWarnings;
+    this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
     this.packageMetadata.explores = outcome.packageMetadata.explores;
     this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
     this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
@@ -280658,6 +281098,11 @@ ${source}` : source;
       } catch (_err) {
         logger.warn(`Could not read manifest for ${packageName}`);
       }
+      const onDiskMaterialization = existingManifest.materialization !== null && typeof existingManifest.materialization === "object" && !Array.isArray(existingManifest.materialization) ? existingManifest.materialization : undefined;
+      const resolvedScope = metadata.scope ?? onDiskMaterialization?.scope ?? existingManifest.scope;
+      const preservedQueryMetadata = metadata.materialization !== undefined && metadata.materialization?.queryMetadata === undefined && onDiskMaterialization?.queryMetadata !== undefined ? { queryMetadata: onDiskMaterialization.queryMetadata } : {};
+      const materializationBase = metadata.materialization !== undefined ? { ...metadata.materialization, ...preservedQueryMetadata } : onDiskMaterialization !== undefined ? { ...onDiskMaterialization } : undefined;
+      const materializationBlock = resolvedScope !== undefined ? { ...materializationBase ?? {}, scope: resolvedScope } : materializationBase;
       const updatedManifest = {
         ...existingManifest,
         name: metadata.name,
@@ -280665,8 +281110,8 @@ ${source}` : source;
         ...metadata.explores !== undefined ? { explores: metadata.explores } : {},
         ...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
         ...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {},
-        ...metadata.scope !== undefined ? { scope: metadata.scope } : {},
-        ...metadata.materialization !== undefined ? { materialization: metadata.materialization } : {}
+        ...resolvedScope !== undefined ? { scope: resolvedScope } : {},
+        ...materializationBlock !== undefined ? { materialization: materializationBlock } : {}
       };
       await fs7.promises.writeFile(manifestPath, JSON.stringify(updatedManifest, null, 2), "utf-8");
       logger.info(`Updated publisher.json for ${packageName}`);
@@ -282469,6 +282914,7 @@ function queryConcurrency() {
 
 // src/controller/materialization.controller.ts
 init_errors();
+var RUN_TRIGGERS = ["publish", "on_demand", "scheduler"];
 
 class MaterializationController {
   materializationService;
@@ -282480,6 +282926,9 @@ class MaterializationController {
   }
   validateCreateBody(body) {
     const result = {};
+    if (body.runContext !== undefined && body.runContext !== null) {
+      result.runContext = this.validateRunContext(body.runContext);
+    }
     if (body.buildInstructions !== undefined && body.buildInstructions !== null) {
       const parsed = this.validateBuildInstructions(body.buildInstructions);
       result.buildInstructions = parsed.sources;
@@ -282504,6 +282953,30 @@ class MaterializationController {
     }
     return result;
   }
+  validateRunContext(raw) {
+    if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+      throw new BadRequestError("runContext must be an object");
+    }
+    const obj = raw;
+    const context = {};
+    if (obj.trigger !== undefined && obj.trigger !== null) {
+      if (typeof obj.trigger !== "string" || !RUN_TRIGGERS.includes(obj.trigger)) {
+        throw new BadRequestError(`runContext.trigger must be one of ${RUN_TRIGGERS.join(" | ")}`);
+      }
+      context.trigger = obj.trigger;
+    }
+    if (obj.runId !== undefined && obj.runId !== null) {
+      if (typeof obj.runId !== "string") {
+        throw new BadRequestError("runContext.runId must be a string");
+      }
+      const violations = queryMetadataViolations({ run_id: obj.runId });
+      if (violations.length > 0) {
+        throw new BadRequestError(`runContext.runId is attached to every statement as the run_id ` + `property: ${violations.join("; ")}`);
+      }
+      context.runId = obj.runId;
+    }
+    return context;
+  }
   validateBuildInstructions(raw) {
     if (typeof raw !== "object" || raw === null) {
       throw new BadRequestError("buildInstructions must be an object");
@@ -285165,7 +285638,7 @@ async function getModelForQuery(environmentStore, environmentName, packageName,
       return { error: details };
     }
     await model.getModel();
-    return { model };
+    return { model, environment };
   } catch (error) {
     let errorDetails;
     if (error instanceof EnvironmentNotFoundError) {
@@ -285424,7 +285897,7 @@ var MAX_RESULT_CHARS = 90000;
 function serialize(envelope) {
   return JSON.stringify(envelope, bigIntReplacer, 2);
 }
-function buildQueryEnvelope(rows, rowLimit, result, renderLogErrors = [], limit = MAX_RESULT_CHARS, rowLimitSource = "server_default") {
+function buildQueryEnvelope(rows, rowLimit, result, renderLogErrors = [], limit = MAX_RESULT_CHARS, rowLimitSource = "server_default", queryCorrelationId = null) {
   const rowCount = Array.isArray(rows) ? rows.length : 0;
   const limitHit = rowLimitSource === "server_default" && rowLimit > 0 && rowCount === rowLimit;
   const envelope = {
@@ -285446,6 +285919,7 @@ function buildQueryEnvelope(rows, rowLimit, result, renderLogErrors = [], limit
     _query_row_limit: rowLimit,
     _limit_source: rowLimitSource,
     _limit_hit: limitHit,
+    ...queryCorrelationId !== null && { _query_id: queryCorrelationId },
     ...renderLogErrors.length > 0 && { renderLogErrors }
   };
   if (limitHit) {
@@ -285529,6 +286003,7 @@ A JSON object, the same shape Credible's execute_query returns, so a data app be
 - _limit_source: "query" when the cap came from the query's own limit:/top:, "server_default" otherwise.
 - _limit_hit: the row count equals that cap AND the cap was the server default.
 - _rows_truncated / _total_rows / _returned_rows: present only when the payload cap dropped rows.
+- _query_id: this query's id in the warehouse's own query history. Present only where enabled.
 - warning, renderLogErrors: present only when they apply.
 
 A query with no limit: of its own gets the server default, so a result landing exactly on _query_row_limit is almost never the whole table. Values above 2^53 are returned as JSON strings so their digits survive.`;
@@ -285558,12 +286033,34 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
     if ("error" in modelResult) {
       return jsonToolError("error://executeQuery/modelAccess", modelResult.error);
     }
-    const { model } = modelResult;
+    const { model, environment } = modelResult;
     logger.info(`[MCP Tool executeQuery] Model found. Proceeding to execute query.`);
     let querySlot = null;
     try {
       querySlot = tryAcquireQuerySlot("mcp:executeQuery");
-      const { result, compactResult, rowLimit, rowLimitSource } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal) : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal), getQueryTimeoutMs());
+      const queryMetadataInput = {
+        environment: environmentName,
+        correlationId: mintCorrelationId(),
+        connectionMetadata: (connectionName) => {
+          try {
+            const connection = environment.getApiConnection(connectionName);
+            return {
+              default: connection.queryMetadata,
+              enforced: connection.queryMetadataEnforced
+            };
+          } catch (error) {
+            logger.debug("[MCP Tool executeQuery] No query-metadata layers for connection", { connectionName, error });
+            return null;
+          }
+        }
+      };
+      const {
+        result,
+        compactResult,
+        rowLimit,
+        rowLimitSource,
+        queryCorrelationId
+      } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal, queryMetadataInput) : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal, queryMetadataInput), getQueryTimeoutMs());
       const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist11(), 1));
       const renderLogs = validateRenderTags2(result);
       const resultUri = buildMalloyUri({
@@ -285572,7 +286069,7 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
         resourceType: "models",
         resourceName: modelPath
       }, "result");
-      const envelope = buildQueryEnvelope(compactResult, rowLimit, result, renderLogs.map((log) => log.message), undefined, rowLimitSource);
+      const envelope = buildQueryEnvelope(compactResult, rowLimit, result, renderLogs.map((log) => log.message), undefined, rowLimitSource, queryCorrelationId);
       const notes = [
         envelope.warning,
         envelope.renderLogErrors && `Render tag problems: ${envelope.renderLogErrors.join("; ")}`
@@ -286038,6 +286535,12 @@ A name is a pointer, not confirmation. A field, source, or view name you saw in
 
 Write Malloy using only the model's names. Load \`skill:malloy-queries\` for syntax (aggregates vs dimensions, joins and field paths, dates, \`where:\` vs \`having:\`, counting) and \`skill:malloy-gotchas-queries\` to avoid the common compile errors. If a model \`view:\` already matches, run it directly rather than rewriting it.
 
+**Check these three before your first \`execute_query\`** - they account for most first-attempt compile failures, and they are the ones a SQL habit gets wrong:
+
+- **Counting.** \`count(field)\` is already the *distinct* count of that field. Malloy has no \`count(distinct field)\`; it is a parse error, not a deprecation.
+- **Separators.** Within a clause, fields are separated by commas or newlines, never \`;\`. A semicolon fails with \`no viable alternative at input '<next-field>'\`.
+- **Join paths.** A dotted path like \`carriers.name\` resolves only if the source declares that join. Confirm the join name and the field under it in a \`get_context\` result instead of inferring either from a table name.
+
 If you define a calculated field that is not already in the model, treat it carefully: ad-hoc definitions are a common source of subtle errors.
 
 - Announce it: tell the user you are adding an ad-hoc field, what it computes, and why the model does not already provide it.
@@ -286056,7 +286559,7 @@ Your first result is a draft, not an answer. The difference between a useful ana
 - **Ground it.** Before interpreting any result, query and state the dataset scope: the time range (\`min\`/\`max\` of the primary date dimension) and the row or entity count. Every number is meaningless without it.
 - **Ask "what would make this wrong?"** then run the query that would expose that problem. A plausible-looking wrong answer is the most dangerous kind.
 - **Check the common failure modes:**
-  - Fan-out / double-counting: if you joined across grain, compare \`count()\` to \`count(distinct key)\`. A large gap means duplication is inflating the aggregates.
+  - Fan-out / double-counting: if you joined across grain, compare \`count()\` to \`count(key)\` - in Malloy \`count(field)\` is already the distinct count. A large gap means duplication is inflating the aggregates.
   - Broken filters: a quick count confirms a filter narrowed the data as expected. Watch case, spelling, and date-format mismatches; a filter that matches nothing still returns a result, just the wrong one.
   - Null-driven loss: \`count() - count(the_field)\` shows how many rows a key field drops.
   - Parts that do not sum to the whole: if you split a total into categories, confirm they add up.
@@ -286634,7 +287137,7 @@ Once the definitions are confirmed, write them into the \`.malloy\` model (see y
 
 ## Output
 
-A confirmed source architecture and a confirmed set of field definitions (renames, dimensions, measures, business decisions), held in the conversation and ready to write into the \`.malloy\` model via your modeling workflow.` }, { name: "malloy-discover", description: "Silent data discovery for Malloy modeling. Used at Step 1 of the modeling workflow. Scans tables, columns, distributions, and relationships without user interaction. The agent builds an internal picture before presenting anything.", body: "# Data Discovery (Step 1, Silent)\n\n> **CRITICAL**: Read the model before writing ANY Malloy code. The model defines the sources, connection names, and fields. Never guess connection names.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n> **PREREQUISITE:** Make sure the Malloy MCP tools (`get_context`, `execute_query`, `search_malloy_docs`) are configured and reachable. If they are not, stop and resolve the MCP connection before continuing.\n\n**This step is silent.** The agent does not present findings to the user yet. That happens in the next step (PROPOSE SCOPE).\n\n## Tools\n\n- **`get_context`**: Ground yourself in the package's sources, views, and fields (with their docs). Call FIRST. The sources and their join paths are the schema you build on.\n- **`execute_query`**: Run ad-hoc queries to preview data, verify values, check NULLs, validate assumptions.\n- **`search_malloy_docs`**: Get Malloy syntax help when needed.\n\n## Workflow\n\n```\n1. Check for prior art signals                 → If found, ask user: \"I found [LookML/dbt] files, use as prior art?\"\n2. If user confirms: read adapter reference        → Follow skill:malloy-lookml-review, keep prior-art notes in-conversation\n3. get_context                           → Ground yourself: sources, views, fields\n4. Inspect source definitions                  → See ALL fields and join paths for key sources\n5. Derive candidate joins/dimensions/measures  → Read them off the model and the data, not a suggestion tool\n6. Define a minimal source if one is missing   → Just enough to run execute_query for previews\n7. execute_query(query)                  → Preview data, verify values, check NULLs, check duplicates\n8. search_malloy_docs(query)                    → Get syntax help when needed\n9. Proceed to Step 2 (PROPOSE SCOPE)\n```\n\n**If the model has no sources defined** and no LookML files are present, do NOT silently retry or proceed without data. Tell the user: \"No model sources were found. Please check that the package points at a connected data source, then try again.\"\n\n**If the model has no sources defined** but LookML files ARE present (LookML-only mode), skip steps 3-7. Use connection name and table paths from the LookML review. Flag all proposals as unvalidated.\n\n**Key principle:** Query data to verify assumptions. Don't ask the user to confirm values you can check yourself.\n\n**Search docs proactively.** If you discover patterns that need derived/pre-aggregated sources, window functions, or unfamiliar features, call `search_malloy_docs` BEFORE writing code, not just when you hit errors.\n\n## Query File for Discovery\n\n**In the schema-first workflow:** Run ad-hoc queries with `execute_query`. If the source you want to preview is not yet defined in the model, define a minimal one against the connection and table so you can run previews. The real model fields are built in later steps.\n\n```malloy\n// minimal source for previewing data during discovery\nsource: explore is my_conn.table('schema.table') extend {}\n```\n\n**In analysis-first mode:** There is no temp file. The analysis `.malloy` file IS your working file. It grows throughout the session and becomes the input for formalizing into a model. See `skill:malloy-analyze` for that workflow.\n\n## What to Capture\n\nWhen reviewing tables and columns, capture:\n\n### Table-Level\n- All tables with row counts\n- Connection name and schema (CRITICAL, never guess)\n- Table roles: fact, dimension, bridge, lookup, staging, operational\n- Join relationships (FK → PK mappings)\n\n### Column-Level\n- Primary key and foreign key columns\n- Data types (watch for string dates, arrays, JSON)\n- Reserved word columns that need backticking (`Date`, `Type`, `number`, `source`, etc.)\n- Column cardinality and NULL rates (via `execute_query`)\n- Data distributions for key numeric and categorical columns\n\n### Data Quality\n- **Check for duplicate rows** on primary keys. Run `group_by: pk, aggregate: count(), having: count() > 1` on each key table. Duplicates cause `sum()` to return nonsensical values.\n- **Denormalized count columns**: beware pre-aggregated fields (e.g., `order_count` in a customer table) that may conflict with joined counts.\n- **Delimited list columns**: flag string columns containing comma-separated values.\n\n### Data-Driven Validation\n\n**Every recommendation must be grounded in queried data, not schema inference.** During discovery, run `execute_query` to validate assumptions before proposing anything in later steps.\n\n| What to validate | Query to run |\n|-----------------|-------------|\n| **Denormalized vs joined values** | Compare pre-computed columns (e.g., `customers.order_count`) against the actual joined aggregate (`count()` from `orders`). Report discrepancy rate. If >0%, flag for user decision. |\n| **Candidate date fields** | When multiple date/timestamp columns exist, query both. What % of rows differ? By how much? This informs which is canonical. |\n| **Numeric column distributions** | Query min, max, avg, percentiles (p25, p50, p75, p95). These inform tier boundaries and detect outliers. |\n| **Categorical column cardinality** | Query distinct values. A `status` column with 5 values behaves differently from one with 500. |\n| **Column usefulness** | Query NULL rates. Columns that are >95% NULL are candidates for `internal`. |\n| **Join cardinality** | Query FK uniqueness: `group_by: fk_col, aggregate: row_count is count(), having: row_count > 1`. Determines `join_one` vs `join_many`. |\n| **Revenue/amount columns** | When multiple money columns exist (`total`, `subtotal`, `amount`, `price`), query a sample to understand how they relate (does `total = subtotal + tax`?). |\n| **Join key value compatibility** | For every proposed join, sample 5-10 actual values from each side. Check for format mismatches: abbreviations (\"4th Av\" vs \"4 Avenue\"), ordinals (\"23 St\" vs \"23rd St\"), casing, prefixes. Mismatched values mean the join won't work even if column names match. |\n| **Mixed-grain rows** | For each key table, run top-N and bottom-N by primary metric. Look for summary/aggregate rows mixed with detail data (e.g., \"System Total\" rows in a station-level table). These corrupt measures if not filtered out. |\n\n**Never assume from column names.** Always query the data to confirm. A column named `total` could include or exclude tax. A `status` column could have unexpected values. A FK could have orphaned references.\n\n### Example Queries\n\n**Tier boundaries**: query distribution, propose breaks from percentiles:\n```malloy\nrun: orders -> {\n  aggregate:\n    min_val is min(sale_price), p25 is sale_price.percentile(25)\n    median_val is sale_price.percentile(50), p75 is sale_price.percentile(75)\n    p95 is sale_price.percentile(95), max_val is max(sale_price)\n}\n```\n\n**Denormalized vs joined**: compare pre-computed column against real aggregate, report match rate:\n```malloy\nrun: customers -> {\n  join_many: orders on customer_id = orders.customer_id\n  aggregate:\n    total is count()\n    match is count() { where: order_count = count(orders.order_id) }\n}\n```\n\n**Canonical date**: when multiple date columns exist, check how often they differ:\n```malloy\nrun: orders -> {\n  aggregate:\n    total is count()\n    same_date is count() { where: created_at::date = submitted_at::date }\n    max_gap_days is max(days(submitted_at - created_at))\n}\n```\n\n**Revenue columns**: when multiple money columns exist, verify their relationship:\n```malloy\nrun: orders -> {\n  aggregate:\n    total_eq_parts is count() { where: abs(sale_price - (subtotal + tax)) < 0.01 }\n    total is count()\n}\n```\n\n### Schema Shape\n- Is this a star/snowflake schema (use base + joined source layers) or normalized/ER-style (may need 3-stage pattern)?\n- Combined vs split tables: prefer filtered/split tables over combined when both exist.\n\n## Computed Source Detection\n\nFlag potential computed sources when:\n\n1. **Grain mismatch**: the analytical scope requires a grain that no physical table provides (e.g., customer-level metrics from an order-grain table)\n2. **Repeated aggregation patterns**: the same GROUP BY + aggregate pattern would be needed in multiple analyses\n3. **Cross-entity aggregations**: the model or the data implies cross-entity aggregations that require a pre-aggregated entity\n\n## Prior Art Detection\n\nCheck for prior art signals at the start of discovery. If a signal is found and the user confirms, **you MUST read** the corresponding reference skill and follow its instructions.\n\n| Signal | Source Type | Reference to Read |\n|--------|------------|-------------------|\n| `.lkml` files in project or subdirectories | lookml | `skill:malloy-lookml-review` |\n| `dbt_project.yml` in project or parent dirs | dbt | dbt review (future) |\n\nThe reference handles inventory, classification, and produces prior-art notes. Keep those notes in-conversation, then continue with normal discovery below.\n\n**If DB connection available (LookML + DB mode):**\n- Read the model and run `execute_query` as normal\n- Use prior art as additional context, not a replacement for data validation\n- **The LookML connection name is NOT the Malloy connection name.** Always use the connection name from the model.\n\n**If no DB connection (LookML-only mode):**\n- Skip the model-read and `execute_query` steps\n- Use connection name and table paths extracted from prior art source files\n- Flag all proposals in Steps 2-4 as **unvalidated**\n- Proceed directly to Step 2 (PROPOSE SCOPE)\n\n**Prior art findings enhance discovery, they don't replace it.** When a DB connection is available, always validate assumptions against the actual data.\n\n## After Discovery\n\nDo NOT present findings to the user yet.\n\n## Done\n\nStep complete. Output: discovery findings (internal: tables, columns, relationships, data quality, prior art). Continue to the next modeling step (see your modeling workflow).\n\n## Verify Source Joins\n\nWhen reading joins off the model or the data, watch for `join_many` where the actual relationship is many-to-one. Always verify cardinality. Prefer `join_one` when each row in the primary table matches at most one row in the joined table." }, { name: "malloy-document", description: 'Add documentation with #(doc) tags to Malloy models so fields and sources are described in plain language. Use when user asks to "add documentation", "add doc tags", "document the model", or wants fields and sources described for natural-language search and discovery. For declaring parameterizable filters with #(filter), see the malloy-model skill. Filters are a runtime/modeling construct (governance, latency, correctness), not a documentation tag.', body: "# Documenting a Malloy Model\n\nAdd `#(doc)` tags to describe sources and fields in plain language so they are easy to find and understand:\n\n| Tag | Purpose | Goes on |\n|-----|---------|---------|\n| `#(doc)` | Plain-language description for natural-language search | source, dimension, measure, view, join |\n| `#(filter)` | Declare a parameterizable filter (runtime/modeling concern, see `malloy-model`) | source |\n\n`#(doc)` is a standard Malloy annotation. It documents a field or source with a human-readable description that downstream tools can surface and search against.\n\n## #(doc) Tag\n\nAdd before any source, dimension, measure, view, or join. When multiple fields share a keyword, use it once as a block header. Tags and field names are indented under the keyword; tags go on the line(s) directly above the field they annotate.\n\n**Tag ordering** (when a field has multiple tags): `#(doc)` → render tags (`# currency`, `# label`, etc.) → field name. Separate each field group with a blank line:\n\n```malloy\n#(doc) Customer who placed the order\njoin_one: users with user_id\n\ndimension:\n  #(doc) Date the order was placed (UTC)\n  order_date is created_at::date\n\nmeasure:\n  #(doc) Total revenue from all orders in USD\n  # currency\n  revenue is sum(total)\n```\n\n### Writing Doc Strings for Retrieval\n\nDoc strings power natural-language search: users type plain-English questions and the system matches against your `#(doc)` strings. Write descriptions that match how analysts would search:\n\n- **Include business meaning**, not code mechanics: what it represents, not how it's implemented\n- **Include units** (USD, count, percentage) and valid values for categorical fields\n- **Avoid Malloy jargon**: never use \"filterable\", \"groupable\", \"dimension\", \"measure\", \"aggregation\"\n\n**Good examples:**\n- `#(doc) Total revenue from completed orders in USD` matches \"what was our revenue?\"\n- `#(doc) Customer signup date (UTC)` matches \"when did the customer join?\"\n- `#(doc) Order status: pending, processing, shipped, delivered, cancelled` matches \"what are the order statuses?\"\n\n**Bad examples:**\n- `#(doc) Filterable dimension for order status`: no analyst searches for \"filterable\"\n- `#(doc) Groupable by region`: \"groupable\" is a system concept\n- `#(doc) Aggregation of total sales`: \"aggregation\" doesn't match natural queries\n\n## #(filter): see `malloy-model`\n\n`#(filter)` is also a `#(...)`-shaped annotation, but unlike `#(doc)` it's a **runtime/modeling construct**: it shapes governance, query latency, and correctness, not discoverability. The full reference (syntax, filter types, `required` / `implicit` flags, and when each applies) lives in `malloy-model` § Parameterizable Filters with `#(filter)` alongside the other source-authoring constructs.\n\nOne rule worth knowing here: filters live on the source, never on the consumer. Ad-hoc reports and notebooks that import a source inherit its filters automatically; they do not (and cannot) declare new ones.\n\n## `internal:` and `private:`: column-level access in a source\n\n`#(doc)` describes what's exposed. Two access modifiers control what's exposed in the first place, and both live **inside** a source's `include {}` block. They are about the source's public API and data sensitivity, not about documentation, so reach for them when curating which columns callers can pick.\n\n| Mechanism | Layer | Why you reach for it |\n|---|---|---|\n| `internal:` | Inside a source (one column in `include {}`) | The column **isn't part of your model's public API**. Common reasons: data is messy (empty/garbage, raw JSON, duplicates), or a documented derived dimension already supersedes it, or the raw column exists only to be joined on / referenced internally and shouldn't appear as a dimension callers can pick. The data may be perfectly fine, it's just not what you want exposed. |\n| `private:` | Inside a source (one column in `include {}`) | The **data is sensitive**: SSN, raw credit card, password. Governance / security concern; a harder block than `internal:`. |\n\nIn one sentence: **`internal:` and `private:` shape what's inside a source's public API; `#(doc)` describes the fields you do expose.**\n\n### Example\n\nA base source pulled from a messy raw table often uses `internal:` to drop raw fields from the public API, while documenting the curated columns with `#(doc)`.\n\n```malloy\n// orders_base.malloy\n#(doc) Raw orders. Use orders.malloy as the entry point for analysis.\nsource: orders_base is conn.table('orders_raw')\n  include {\n    public: id, customer_id, order_date, total\n    internal: raw_json_payload, deprecated_status_code, _temp_dedup_marker\n  }\n  extend {\n    primary_key: id\n  }\n```\n\n```malloy\n// orders.malloy\nimport \"orders_base.malloy\"\n\n#(doc) Order analysis. Use for revenue, fulfillment, and customer-order joins.\nsource: orders is orders_base extend {\n  // joins, measures, curated dimensions\n}\n```\n\nThe base source stays fully queryable (`run: orders_base -> { ... }` still works); `internal:` only governs which columns appear as public dimensions callers can pick.\n\n## Annotating Columns in Include (Experimental)\n\nWith `##! experimental.access_modifiers`, you can add `#(doc)` tags to raw table columns inside `include` blocks. This documents columns without redefining them as dimensions.\n\n```malloy\n##! experimental.access_modifiers\n\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order line item identifier\n    id\n\n    #(doc) Customer email address\n    email\n\n    #(doc) Order status: pending, shipped, delivered\n    status\n\n  // internal: only for verified noise (empty cols, raw JSON blobs, duplicates)\n}\nextend {\n  // ... dimensions and measures\n}\n```\n\n**When to use:**\n- Documenting raw columns without creating explicit dimensions\n- Curating which columns are public vs internal\n\n## Source-Level Documentation\n\nDocument **when to use** a source, not what it contains. Dimensions and measures can already be searched directly, so the source-level `#(doc)` should describe what questions/analyses this source answers.\n\n**Base source files:** Document what the table represents.\n```malloy\n#(doc) Customer records with demographics and segmentation. One row per customer.\nsource: customers is conn.table('sales.customers') extend { ... }\n```\n\n**Source files:** Document what analytical questions the source answers.\n```malloy\n#(doc) Customer health analysis. Use for retention, segmentation, churn risk, and lifetime value. For order-level analysis, use order_analysis instead.\nsource: customer_health is customers extend { ... }\n```\n\n**Best practices:**\n- Add `#(doc)` to all base source and joined source definitions\n- Base source docs: describe what the table is (one row per what)\n- Source docs: describe what questions/analyses the source answers\n- Documentation happens per-source-file, not in one monolithic file\n\n## Flag Ambiguous Descriptions\n\nAfter writing `#(doc)` tags, present any that required judgment to the user for confirmation:\n\n| Field | Proposed doc | Confidence | Uncertainty |\n|-------|-------------|------------|-------------|\n| `total` | \"Total order amount in USD\" | Medium | Could be gross or net, verified with sample query |\n| `status` | \"Order status: pending, shipped, delivered\" | High | Values confirmed via a query of distinct values |\n\nOnly flag fields where the description required assumptions about business meaning, units, or valid values. When in doubt about valid values, run a quick query against the data to confirm them before writing the description. Use `malloy_getContext` to ground yourself in the package's sources and fields and `malloy_executeQuery` to check distinct values, for example `run: source -> { group_by: status }`.\n\n## Done\n\nStep complete. Output: `#(doc)` tags added to all public fields and sources." }, { name: "malloy-getting-started", description: "First steps for using a Malloy Publisher deployment through its MCP tools. Use when connecting to Publisher for the first time, when you do not yet know the available environments, packages, or models, or when a user asks what data they can explore. Covers verifying the server, discovering data with malloy_getContext, and running a first grounded query.", body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.' }, { name: "malloy-gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: Only `seconds` / `minutes` / `hours` / `days`\n\n`weeks()`, `months()`, `quarters()`, `years()` are **documented but don't work** in this build; only `seconds`, `minutes`, `hours`, `days` actually function. Compute in days and derive the larger unit: a *units conversion*, not a calendar-floored duration:\n\n```malloy\n// WRONG: weeks()/months() don't compile\ndimension: weeks_open is weeks(opened_at to closed_at)\n\n// RIGHT: measure in days, convert (documents that it's approximate)\ndimension: days_open  is days(opened_at to closed_at)\ndimension: weeks_open is days(opened_at to closed_at) / 7      // ≈ weeks\ndimension: months_open is days(opened_at to closed_at) / 30.44 // ≈ months\n```\n\n(Contrast: `search_malloy_docs` gets this right when asked narrowly; trust the docs on the supported units, not on the missing ones.)\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `seconds()`): only `seconds`/`minutes`/`hours`/`days` exist (see above)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "malloy-gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`\n\n## Fields Within a Clause: Commas or Newlines, Never Semicolons\n\nSemicolons are not a separator anywhere in Malloy. Multiple fields under one `aggregate:` / `group_by:` are separated by commas (inline) or newlines (one per line); a `;` fails with `no viable alternative at input '<next-field>'` pointing at the field right after it.\n\n```malloy\n// WRONG: semicolons between fields\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT: commas inline...\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n// ...or newlines\nrun: schools -> {\n  aggregate:\n    total is count()\n    charters is count() { where: is_charter }\n}\n```" }, { name: "malloy-gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
+A confirmed source architecture and a confirmed set of field definitions (renames, dimensions, measures, business decisions), held in the conversation and ready to write into the \`.malloy\` model via your modeling workflow.` }, { name: "malloy-discover", description: "Silent data discovery for Malloy modeling. Used at Step 1 of the modeling workflow. Scans tables, columns, distributions, and relationships without user interaction. The agent builds an internal picture before presenting anything.", body: "# Data Discovery (Step 1, Silent)\n\n> **CRITICAL**: Read the model before writing ANY Malloy code. The model defines the sources, connection names, and fields. Never guess connection names.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n> **PREREQUISITE:** Make sure the Malloy MCP tools (`get_context`, `execute_query`, `search_malloy_docs`) are configured and reachable. If they are not, stop and resolve the MCP connection before continuing.\n\n**This step is silent.** The agent does not present findings to the user yet. That happens in the next step (PROPOSE SCOPE).\n\n## Tools\n\n- **`get_context`**: Ground yourself in the package's sources, views, and fields (with their docs). Call FIRST. The sources and their join paths are the schema you build on.\n- **`execute_query`**: Run ad-hoc queries to preview data, verify values, check NULLs, validate assumptions.\n- **`search_malloy_docs`**: Get Malloy syntax help when needed.\n\n## Workflow\n\n```\n1. Check for prior art signals                 → If found, ask user: \"I found [LookML/dbt] files, use as prior art?\"\n2. If user confirms: read adapter reference        → Follow skill:malloy-lookml-review, keep prior-art notes in-conversation\n3. get_context                           → Ground yourself: sources, views, fields\n4. Inspect source definitions                  → See ALL fields and join paths for key sources\n5. Derive candidate joins/dimensions/measures  → Read them off the model and the data, not a suggestion tool\n6. Define a minimal source if one is missing   → Just enough to run execute_query for previews\n7. execute_query(query)                  → Preview data, verify values, check NULLs, check duplicates\n8. search_malloy_docs(query)                    → Get syntax help when needed\n9. Proceed to Step 2 (PROPOSE SCOPE)\n```\n\n**If the model has no sources defined** and no LookML files are present, do NOT silently retry or proceed without data. Tell the user: \"No model sources were found. Please check that the package points at a connected data source, then try again.\"\n\n**If the model has no sources defined** but LookML files ARE present (LookML-only mode), skip steps 3-7. Use connection name and table paths from the LookML review. Flag all proposals as unvalidated.\n\n**Key principle:** Query data to verify assumptions. Don't ask the user to confirm values you can check yourself.\n\n**Search docs proactively.** If you discover patterns that need derived/pre-aggregated sources, window functions, or unfamiliar features, call `search_malloy_docs` BEFORE writing code, not just when you hit errors.\n\n## Query File for Discovery\n\n**In the schema-first workflow:** Run ad-hoc queries with `execute_query`. If the source you want to preview is not yet defined in the model, define a minimal one against the connection and table so you can run previews. The real model fields are built in later steps.\n\n```malloy\n// minimal source for previewing data during discovery\nsource: explore is my_conn.table('schema.table') extend {}\n```\n\n**In analysis-first mode:** There is no temp file. The analysis `.malloy` file IS your working file. It grows throughout the session and becomes the input for formalizing into a model. See `skill:malloy-analyze` for that workflow.\n\n## What to Capture\n\nWhen reviewing tables and columns, capture:\n\n### Table-Level\n- All tables with row counts\n- Connection name and schema (CRITICAL, never guess)\n- Table roles: fact, dimension, bridge, lookup, staging, operational\n- Join relationships (FK → PK mappings)\n\n### Column-Level\n- Primary key and foreign key columns\n- Data types (watch for string dates, arrays, JSON)\n- Reserved word columns that need backticking (`Date`, `Type`, `number`, `source`, etc.)\n- Column cardinality and NULL rates (via `execute_query`)\n- Data distributions for key numeric and categorical columns\n\n### Data Quality\n- **Check for duplicate rows** on primary keys. Run `group_by: pk, aggregate: count(), having: count() > 1` on each key table. Duplicates cause `sum()` to return nonsensical values.\n- **Denormalized count columns**: beware pre-aggregated fields (e.g., `order_count` in a customer table) that may conflict with joined counts.\n- **Delimited list columns**: flag string columns containing comma-separated values.\n\n### Data-Driven Validation\n\n**Every recommendation must be grounded in queried data, not schema inference.** During discovery, run `execute_query` to validate assumptions before proposing anything in later steps.\n\n| What to validate | Query to run |\n|-----------------|-------------|\n| **Denormalized vs joined values** | Compare pre-computed columns (e.g., `customers.order_count`) against the actual joined aggregate (`count()` from `orders`). Report discrepancy rate. If >0%, flag for user decision. |\n| **Candidate date fields** | When multiple date/timestamp columns exist, query both. What % of rows differ? By how much? This informs which is canonical. |\n| **Numeric column distributions** | Query min, max, avg, percentiles (p25, p50, p75, p95). These inform tier boundaries and detect outliers. |\n| **Categorical column cardinality** | Query distinct values. A `status` column with 5 values behaves differently from one with 500. |\n| **Column usefulness** | Query NULL rates. Columns that are >95% NULL are candidates for `internal`. |\n| **Join cardinality** | Query FK uniqueness: `group_by: fk_col, aggregate: row_count is count(), having: row_count > 1`. Determines `join_one` vs `join_many`. |\n| **Revenue/amount columns** | When multiple money columns exist (`total`, `subtotal`, `amount`, `price`), query a sample to understand how they relate (does `total = subtotal + tax`?). |\n| **Join key value compatibility** | For every proposed join, sample 5-10 actual values from each side. Check for format mismatches: abbreviations (\"4th Av\" vs \"4 Avenue\"), ordinals (\"23 St\" vs \"23rd St\"), casing, prefixes. Mismatched values mean the join won't work even if column names match. |\n| **Mixed-grain rows** | For each key table, run top-N and bottom-N by primary metric. Look for summary/aggregate rows mixed with detail data (e.g., \"System Total\" rows in a station-level table). These corrupt measures if not filtered out. |\n\n**Never assume from column names.** Always query the data to confirm. A column named `total` could include or exclude tax. A `status` column could have unexpected values. A FK could have orphaned references.\n\n### Example Queries\n\n**Tier boundaries**: query distribution, propose breaks from percentiles:\n```malloy\nrun: orders -> {\n  aggregate:\n    min_val is min(sale_price), p25 is sale_price.percentile(25)\n    median_val is sale_price.percentile(50), p75 is sale_price.percentile(75)\n    p95 is sale_price.percentile(95), max_val is max(sale_price)\n}\n```\n\n**Denormalized vs joined**: compare pre-computed column against real aggregate, report match rate:\n```malloy\nrun: customers -> {\n  join_many: orders on customer_id = orders.customer_id\n  aggregate:\n    total is count()\n    match is count() { where: order_count = count(orders.order_id) }\n}\n```\n\n**Canonical date**: when multiple date columns exist, check how often they differ:\n```malloy\nrun: orders -> {\n  aggregate:\n    total is count()\n    same_date is count() { where: created_at::date = submitted_at::date }\n    max_gap_days is max(days(submitted_at - created_at))\n}\n```\n\n**Revenue columns**: when multiple money columns exist, verify their relationship:\n```malloy\nrun: orders -> {\n  aggregate:\n    total_eq_parts is count() { where: abs(sale_price - (subtotal + tax)) < 0.01 }\n    total is count()\n}\n```\n\n### Schema Shape\n- Is this a star/snowflake schema (use base + joined source layers) or normalized/ER-style (may need 3-stage pattern)?\n- Combined vs split tables: prefer filtered/split tables over combined when both exist.\n\n## Computed Source Detection\n\nFlag potential computed sources when:\n\n1. **Grain mismatch**: the analytical scope requires a grain that no physical table provides (e.g., customer-level metrics from an order-grain table)\n2. **Repeated aggregation patterns**: the same GROUP BY + aggregate pattern would be needed in multiple analyses\n3. **Cross-entity aggregations**: the model or the data implies cross-entity aggregations that require a pre-aggregated entity\n\n## Prior Art Detection\n\nCheck for prior art signals at the start of discovery. If a signal is found and the user confirms, **you MUST read** the corresponding reference skill and follow its instructions.\n\n| Signal | Source Type | Reference to Read |\n|--------|------------|-------------------|\n| `.lkml` files in project or subdirectories | lookml | `skill:malloy-lookml-review` |\n| `dbt_project.yml` in project or parent dirs | dbt | dbt review (future) |\n\nThe reference handles inventory, classification, and produces prior-art notes. Keep those notes in-conversation, then continue with normal discovery below.\n\n**If DB connection available (LookML + DB mode):**\n- Read the model and run `execute_query` as normal\n- Use prior art as additional context, not a replacement for data validation\n- **The LookML connection name is NOT the Malloy connection name.** Always use the connection name from the model.\n\n**If no DB connection (LookML-only mode):**\n- Skip the model-read and `execute_query` steps\n- Use connection name and table paths extracted from prior art source files\n- Flag all proposals in Steps 2-4 as **unvalidated**\n- Proceed directly to Step 2 (PROPOSE SCOPE)\n\n**Prior art findings enhance discovery, they don't replace it.** When a DB connection is available, always validate assumptions against the actual data.\n\n## After Discovery\n\nDo NOT present findings to the user yet.\n\n## Done\n\nStep complete. Output: discovery findings (internal: tables, columns, relationships, data quality, prior art). Continue to the next modeling step (see your modeling workflow).\n\n## Verify Source Joins\n\nWhen reading joins off the model or the data, watch for `join_many` where the actual relationship is many-to-one. Always verify cardinality. Prefer `join_one` when each row in the primary table matches at most one row in the joined table." }, { name: "malloy-document", description: 'Add documentation with #(doc) tags to Malloy models so fields and sources are described in plain language. Use when user asks to "add documentation", "add doc tags", "document the model", or wants fields and sources described for natural-language search and discovery. For declaring parameterizable filters with #(filter), see the malloy-model skill. Filters are a runtime/modeling construct (governance, latency, correctness), not a documentation tag.', body: "# Documenting a Malloy Model\n\nAdd `#(doc)` tags to describe sources and fields in plain language so they are easy to find and understand:\n\n| Tag | Purpose | Goes on |\n|-----|---------|---------|\n| `#(doc)` | Plain-language description for natural-language search | source, dimension, measure, view, join |\n| `#(filter)` | Declare a parameterizable filter (runtime/modeling concern, see `malloy-model`) | source |\n\n`#(doc)` is a standard Malloy annotation. It documents a field or source with a human-readable description that downstream tools can surface and search against.\n\n## #(doc) Tag\n\nAdd before any source, dimension, measure, view, or join. When multiple fields share a keyword, use it once as a block header. Tags and field names are indented under the keyword; tags go on the line(s) directly above the field they annotate.\n\n**Tag ordering** (when a field has multiple tags): `#(doc)` → render tags (`# currency`, `# label`, etc.) → field name. Separate each field group with a blank line:\n\n```malloy\n#(doc) Customer who placed the order\njoin_one: users with user_id\n\ndimension:\n  #(doc) Date the order was placed (UTC)\n  order_date is created_at::date\n\nmeasure:\n  #(doc) Total revenue from all orders in USD\n  # currency\n  revenue is sum(total)\n```\n\n### Writing Doc Strings for Retrieval\n\nDoc strings power natural-language search: users type plain-English questions and the system matches against your `#(doc)` strings. Write descriptions that match how analysts would search:\n\n- **Include business meaning**, not code mechanics: what it represents, not how it's implemented\n- **Include units** (USD, count, percentage) and valid values for categorical fields\n- **Avoid Malloy jargon**: never use \"filterable\", \"groupable\", \"dimension\", \"measure\", \"aggregation\"\n\n**Good examples:**\n- `#(doc) Total revenue from completed orders in USD` matches \"what was our revenue?\"\n- `#(doc) Customer signup date (UTC)` matches \"when did the customer join?\"\n- `#(doc) Order status: pending, processing, shipped, delivered, cancelled` matches \"what are the order statuses?\"\n\n**Bad examples:**\n- `#(doc) Filterable dimension for order status`: no analyst searches for \"filterable\"\n- `#(doc) Groupable by region`: \"groupable\" is a system concept\n- `#(doc) Aggregation of total sales`: \"aggregation\" doesn't match natural queries\n\n## #(filter): see `malloy-model`\n\n`#(filter)` is also a `#(...)`-shaped annotation, but unlike `#(doc)` it's a **runtime/modeling construct**: it shapes governance, query latency, and correctness, not discoverability. The full reference (syntax, filter types, `required` / `implicit` flags, and when each applies) lives in `malloy-model` § Parameterizable Filters with `#(filter)` alongside the other source-authoring constructs.\n\nOne rule worth knowing here: filters live on the source, never on the consumer. Ad-hoc reports and notebooks that import a source inherit its filters automatically; they do not (and cannot) declare new ones.\n\n## `internal:` and `private:`: column-level access in a source\n\n`#(doc)` describes what's exposed. Two access modifiers control what's exposed in the first place, and both live **inside** a source's `include {}` block. They are about the source's public API and data sensitivity, not about documentation, so reach for them when curating which columns callers can pick.\n\n| Mechanism | Layer | Why you reach for it |\n|---|---|---|\n| `internal:` | Inside a source (one column in `include {}`) | The column **isn't part of your model's public API**. Common reasons: data is messy (empty/garbage, raw JSON, duplicates), or a documented derived dimension already supersedes it, or the raw column exists only to be joined on / referenced internally and shouldn't appear as a dimension callers can pick. The data may be perfectly fine, it's just not what you want exposed. |\n| `private:` | Inside a source (one column in `include {}`) | The **data is sensitive**: SSN, raw credit card, password. Governance / security concern; a harder block than `internal:`. |\n\nIn one sentence: **`internal:` and `private:` shape what's inside a source's public API; `#(doc)` describes the fields you do expose.**\n\n### Example\n\nA base source pulled from a messy raw table often uses `internal:` to drop raw fields from the public API, while documenting the curated columns with `#(doc)`.\n\n```malloy\n// orders_base.malloy\n#(doc) Raw orders. Use orders.malloy as the entry point for analysis.\nsource: orders_base is conn.table('orders_raw')\n  include {\n    public: id, customer_id, order_date, total\n    internal: raw_json_payload, deprecated_status_code, _temp_dedup_marker\n  }\n  extend {\n    primary_key: id\n  }\n```\n\n```malloy\n// orders.malloy\nimport \"orders_base.malloy\"\n\n#(doc) Order analysis. Use for revenue, fulfillment, and customer-order joins.\nsource: orders is orders_base extend {\n  // joins, measures, curated dimensions\n}\n```\n\nThe base source stays fully queryable (`run: orders_base -> { ... }` still works); `internal:` only governs which columns appear as public dimensions callers can pick.\n\n## Annotating Columns in Include (Experimental)\n\nWith `##! experimental.access_modifiers`, you can add `#(doc)` tags to raw table columns inside `include` blocks. This documents columns without redefining them as dimensions.\n\n```malloy\n##! experimental.access_modifiers\n\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order line item identifier\n    id\n\n    #(doc) Customer email address\n    email\n\n    #(doc) Order status: pending, shipped, delivered\n    status\n\n  // internal: only for verified noise (empty cols, raw JSON blobs, duplicates)\n}\nextend {\n  // ... dimensions and measures\n}\n```\n\n**When to use:**\n- Documenting raw columns without creating explicit dimensions\n- Curating which columns are public vs internal\n\n## Source-Level Documentation\n\nDocument **when to use** a source, not what it contains. Dimensions and measures can already be searched directly, so the source-level `#(doc)` should describe what questions/analyses this source answers.\n\n**Base source files:** Document what the table represents.\n```malloy\n#(doc) Customer records with demographics and segmentation. One row per customer.\nsource: customers is conn.table('sales.customers') extend { ... }\n```\n\n**Source files:** Document what analytical questions the source answers.\n```malloy\n#(doc) Customer health analysis. Use for retention, segmentation, churn risk, and lifetime value. For order-level analysis, use order_analysis instead.\nsource: customer_health is customers extend { ... }\n```\n\n**Best practices:**\n- Add `#(doc)` to all base source and joined source definitions\n- Base source docs: describe what the table is (one row per what)\n- Source docs: describe what questions/analyses the source answers\n- Documentation happens per-source-file, not in one monolithic file\n\n## Flag Ambiguous Descriptions\n\nAfter writing `#(doc)` tags, present any that required judgment to the user for confirmation:\n\n| Field | Proposed doc | Confidence | Uncertainty |\n|-------|-------------|------------|-------------|\n| `total` | \"Total order amount in USD\" | Medium | Could be gross or net, verified with sample query |\n| `status` | \"Order status: pending, shipped, delivered\" | High | Values confirmed via a query of distinct values |\n\nOnly flag fields where the description required assumptions about business meaning, units, or valid values. When in doubt about valid values, run a quick query against the data to confirm them before writing the description. Use `malloy_getContext` to ground yourself in the package's sources and fields and `malloy_executeQuery` to check distinct values, for example `run: source -> { group_by: status }`.\n\n## Done\n\nStep complete. Output: `#(doc)` tags added to all public fields and sources." }, { name: "malloy-getting-started", description: "First steps for using a Malloy Publisher deployment through its MCP tools. Use when connecting to Publisher for the first time, when you do not yet know the available environments, packages, or models, or when a user asks what data they can explore. Covers verifying the server, discovering data with malloy_getContext, and running a first grounded query.", body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.' }, { name: "malloy-gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: Only `seconds` / `minutes` / `hours` / `days`\n\n`weeks()`, `months()`, `quarters()`, `years()` are **documented but don't work** in this build; only `seconds`, `minutes`, `hours`, `days` actually function. Compute in days and derive the larger unit: a *units conversion*, not a calendar-floored duration:\n\n```malloy\n// WRONG: weeks()/months() don't compile\ndimension: weeks_open is weeks(opened_at to closed_at)\n\n// RIGHT: measure in days, convert (documents that it's approximate)\ndimension: days_open  is days(opened_at to closed_at)\ndimension: weeks_open is days(opened_at to closed_at) / 7      // ≈ weeks\ndimension: months_open is days(opened_at to closed_at) / 30.44 // ≈ months\n```\n\n(Contrast: `search_malloy_docs` gets this right when asked narrowly; trust the docs on the supported units, not on the missing ones.)\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `seconds()`): only `seconds`/`minutes`/`hours`/`days` exist (see above)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "malloy-gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n//   \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`\n\n## Fields Within a Clause: Commas or Newlines, Never Semicolons\n\nSemicolons are not a separator anywhere in Malloy. Multiple fields under one `aggregate:` / `group_by:` are separated by commas (inline) or newlines (one per line); a `;` fails with `no viable alternative at input '<next-field>'` pointing at the field right after it.\n\n```malloy\n// WRONG: semicolons between fields\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT: commas inline...\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n// ...or newlines\nrun: schools -> {\n  aggregate:\n    total is count()\n    charters is count() { where: is_charter }\n}\n```" }, { name: "malloy-gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
 
 > **Read this before adding renderer annotations.** These patterns cause most rendering issues.
 
@@ -287556,7 +288059,7 @@ Present coverage in this order:
 5. **Deferred items**: list everything marked \`deferred\` across sources and fields, with rationale, so the user knows what's available for future iterations
 6. **Full field coverage tables** (Section 2 detail): present per base source if the user wants to drill in
 
-**Ask the user:** "Does this coverage look right? Anything deferred that you'd like to add now, or anything modeled that should be removed?"` }, { name: "malloy-materialization", description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.", body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n   - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n   - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"scope\": \"package\",  // default; \"version\" = each published version owns its own tables\n     \"materialization\": { \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" } }\n   }\n   ```\n   Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n   - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope.\n   - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n   - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself." }, { name: "malloy-materialization-tuning", description: "Optimize a package's Malloy Persistence materializations for cost and performance using the malloy-pub CLI and the materialization history. Recommend what to persist, what to stop persisting, and how to schedule/scope it. Use when the user asks to make a package cheaper or faster, tune persistence, decide what to materialize, or review persist/schedule choices.", body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `malloy_getContext` the package) to list them.\n- **Schedule + scope:**\n\n  ```bash\n  malloy-pub schedule view --environment <env> --package <pkg>\n  ```\n\n  This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **Across the whole environment** (all packages, newest first; the rows are interleaved and labeled by package, not grouped into contiguous per-package blocks):\n\n  ```bash\n  malloy-pub list materialization --environment <env>\n  ```\n\n  Columns: Package, ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **For one package:**\n\n  ```bash\n  malloy-pub list materialization --environment <env> --package <pkg>\n  ```\n\n- **A single run's detail** (the cost signals):\n\n  ```bash\n  malloy-pub get materialization <id> --environment <env> --package <pkg>\n  ```\n\n  In the JSON, read:\n  - `metadata.durationMs`: how long the build took.\n  - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n  - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n  - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly.\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n  ```bash\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n  ```bash\n  malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n  > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n  ```bash\n  malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg>   # 5-field UTC cron\n  malloy-pub schedule clear --environment <env> --package <pkg>\n  ```\n\n  `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared at the root of the package's `publisher.json` (`\"scope\": \"package\"` or `\"version\"`). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters." }, { name: "malloy-model", description: 'Build Malloy semantic models with base source and joined source files. Use when creating or modifying .malloy files, user asks to "create a malloy model", "add dimensions", "add measures", "create a source", or any Malloy model authoring task.', body: `# Building Malloy Models
+**Ask the user:** "Does this coverage look right? Anything deferred that you'd like to add now, or anything modeled that should be removed?"` }, { name: "malloy-materialization", description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.", body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n   - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n   - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"materialization\": {\n       \"scope\": \"package\",  // default; \"version\" = each published version owns its own tables\n       \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n       \"queryMetadata\": { \"team\": \"finance\" }  // tags the build's backend statements\n     }\n   }\n   ```\n   Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n   - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n   - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n   - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n   - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself." }, { name: "malloy-materialization-tuning", description: "Optimize a package's Malloy Persistence materializations for cost and performance using the malloy-pub CLI and the materialization history. Recommend what to persist, what to stop persisting, and how to schedule/scope it. Use when the user asks to make a package cheaper or faster, tune persistence, decide what to materialize, or review persist/schedule choices.", body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `malloy_getContext` the package) to list them.\n- **Schedule + scope:**\n\n  ```bash\n  malloy-pub schedule view --environment <env> --package <pkg>\n  ```\n\n  This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **Across the whole environment** (all packages, newest first; the rows are interleaved and labeled by package, not grouped into contiguous per-package blocks):\n\n  ```bash\n  malloy-pub list materialization --environment <env>\n  ```\n\n  Columns: Package, ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **For one package:**\n\n  ```bash\n  malloy-pub list materialization --environment <env> --package <pkg>\n  ```\n\n- **A single run's detail** (the cost signals):\n\n  ```bash\n  malloy-pub get materialization <id> --environment <env> --package <pkg>\n  ```\n\n  In the JSON, read:\n  - `metadata.durationMs`: how long the build took.\n  - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n  - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n  - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly.\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n  ```bash\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n  ```bash\n  malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n  malloy-pub materialize --environment <env> --package <pkg> --wait\n  ```\n\n  > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n  ```bash\n  malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg>   # 5-field UTC cron\n  malloy-pub schedule clear --environment <env> --package <pkg>\n  ```\n\n  `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters." }, { name: "malloy-model", description: 'Build Malloy semantic models with base source and joined source files. Use when creating or modifying .malloy files, user asks to "create a malloy model", "add dimensions", "add measures", "create a source", or any Malloy model authoring task.', body: `# Building Malloy Models
 
 > **Tool names** are written bare here - \`get_context\`, \`execute_query\`, \`search_malloy_docs\`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
 
@@ -289201,6 +289704,151 @@ function initializeMcpServer(environmentStore) {
   return mcpServer;
 }
 
+// src/mcp_config.ts
+import * as fs9 from "fs";
+import * as os3 from "os";
+import * as path11 from "path";
+init_logger();
+var MCP_CONFIG_FILENAME = ".mcp.json";
+function malloyServer(endpoint) {
+  return { type: "http", url: endpoint };
+}
+function resolveBoundPort(address, requestedPort) {
+  return typeof address === "object" && address ? address.port : requestedPort;
+}
+function resolveClientHost(address, fallbackHost) {
+  const usableFallback = /^[0-9a-fA-F:.]+$/.test(fallbackHost) ? fallbackHost : "";
+  const fromSocket = typeof address === "object" && address ? address.address : undefined;
+  const raw = typeof fromSocket === "string" && fromSocket ? fromSocket : usableFallback;
+  if (raw === "0.0.0.0" || raw === "")
+    return "127.0.0.1";
+  if (raw === "::" || raw === "::0")
+    return "[::1]";
+  if (!raw.includes(":"))
+    return raw;
+  return `[${raw.split("%")[0]}]`;
+}
+function mcpEndpoint(host, port) {
+  return `http://${host}:${port}/mcp`;
+}
+function findGitWorkTreeRoot(dir) {
+  let current = path11.resolve(dir);
+  for (;; ) {
+    if (fs9.existsSync(path11.join(current, ".git")))
+      return current;
+    const parent = path11.dirname(current);
+    if (parent === current)
+      return;
+    current = parent;
+  }
+}
+function mcpConfigEnabled() {
+  return !(parseBoolEnv("PUBLISHER_NO_MCP_CONFIG") ?? false);
+}
+function ensureMcpConfig(options) {
+  const { dir, endpoint, requestedPort, boundPort, homeDir } = options;
+  const file = path11.join(dir, MCP_CONFIG_FILENAME);
+  try {
+    const existing = (() => {
+      try {
+        return {
+          path: file,
+          isSymlink: fs9.lstatSync(file).isSymbolicLink()
+        };
+      } catch {
+        return;
+      }
+    })();
+    const staleConfig = existing?.path;
+    if (boundPort !== requestedPort) {
+      return {
+        action: "skipped-unstable-port",
+        dir,
+        endpoint,
+        requestedPort,
+        boundPort,
+        staleConfig
+      };
+    }
+    const realish = (p) => {
+      try {
+        return fs9.realpathSync(p);
+      } catch {
+        return path11.resolve(p);
+      }
+    };
+    if (realish(dir) === realish(homeDir ?? os3.homedir())) {
+      return { action: "skipped-home", dir, endpoint, staleConfig };
+    }
+    if (path11.resolve(dir) === path11.parse(path11.resolve(dir)).root) {
+      return { action: "skipped-root", dir, endpoint, staleConfig };
+    }
+    const gitRoot = findGitWorkTreeRoot(dir);
+    if (gitRoot !== undefined) {
+      const rootCandidate = path11.join(gitRoot, MCP_CONFIG_FILENAME);
+      return {
+        action: "skipped-git",
+        dir,
+        gitRoot,
+        rootConfig: fs9.existsSync(rootCandidate) ? rootCandidate : undefined,
+        endpoint,
+        staleConfig
+      };
+    }
+    if (existing?.isSymlink)
+      return { action: "exists", file, endpoint };
+    const body = JSON.stringify({ mcpServers: { malloy: malloyServer(endpoint) } }, null, 2) + `
+`;
+    fs9.writeFileSync(file, body, { encoding: "utf8", flag: "wx" });
+    return { action: "created", file };
+  } catch (error) {
+    const code = error?.code;
+    if (code === "EEXIST")
+      return { action: "exists", file, endpoint };
+    return {
+      action: "failed",
+      file,
+      problem: error instanceof Error ? error.message : String(error),
+      endpoint
+    };
+  }
+}
+function addCommand(endpoint) {
+  return `claude mcp add --transport http malloy '${endpoint}'`;
+}
+function staleNote(staleConfig, alreadyNamed) {
+  return staleConfig === undefined || staleConfig === alreadyNamed ? "" : `. Note that ${staleConfig} is already here and was not written by this run, so an agent started here uses whatever that names.`;
+}
+function logMcpConfigOutcome(outcome) {
+  switch (outcome.action) {
+    case "created":
+      logger.info(`Wrote ${outcome.file} so an agent started in this directory finds this server. Disable with --no-mcp-config.`);
+      return;
+    case "exists":
+      logger.info(`Left the existing ${outcome.file} alone, unread. This server is at ${outcome.endpoint}. If an agent started here reaches a different Publisher than you expect, ask it to run malloy_getContext, which names the environment and packages it is actually talking to. To point it here: ${addCommand(outcome.endpoint)}`);
+      return;
+    case "failed":
+      logger.info(`Could not write ${outcome.file} (${outcome.problem}). An agent started here will not find this server on its own. To connect one, run: ${addCommand(outcome.endpoint)}`);
+      return;
+    case "skipped-git":
+      logger.info(outcome.rootConfig !== undefined ? `Did not write ${MCP_CONFIG_FILENAME} into ${outcome.dir} because it is inside the git working tree at ${outcome.gitRoot}, which already has ${outcome.rootConfig}. This server is at ${outcome.endpoint}; if your agent does not list malloy, run this from the directory you start it in: ${addCommand(outcome.endpoint)}${staleNote(outcome.staleConfig, outcome.rootConfig)}` : `Did not write ${MCP_CONFIG_FILENAME} into ${outcome.dir} because it is inside the git working tree at ${outcome.gitRoot}. To connect an agent, run this from the directory you start it in: ${addCommand(outcome.endpoint)}${staleNote(outcome.staleConfig)}`);
+      return;
+    case "skipped-home":
+      logger.info(`Did not write ${MCP_CONFIG_FILENAME} into your home directory (${outcome.dir}). To connect an agent, run this from the directory you start it in: ${addCommand(outcome.endpoint)}${staleNote(outcome.staleConfig)}`);
+      return;
+    case "skipped-unstable-port":
+      logger.info(`Did not write ${MCP_CONFIG_FILENAME}: this server asked for MCP port ${outcome.requestedPort} and got ${outcome.boundPort}, so the port changes from run to run and a saved config would be wrong next boot. To connect an agent to this run: ${addCommand(outcome.endpoint)}. That registration outlives this run, and the port will have moved by the next one, so undo it with claude mcp remove malloy${staleNote(outcome.staleConfig)}`);
+      return;
+    case "skipped-root":
+      logger.info(`Did not write ${MCP_CONFIG_FILENAME} into the filesystem root (${outcome.dir}). To connect an agent, run this from the directory you start it in: ${addCommand(outcome.endpoint)}${staleNote(outcome.staleConfig)}`);
+      return;
+    default: {
+      const exhaustive = outcome;
+      return exhaustive;
+    }
+  }
+}
+
 // src/server-old.ts
 init_errors();
 init_logger();
@@ -289876,8 +290524,8 @@ import {
   Runtime as Runtime4
 } from "@malloydata/malloy";
 import { mkdtempSync, rmSync } from "node:fs";
-import os3 from "node:os";
-import path11 from "node:path";
+import os4 from "node:os";
+import path12 from "node:path";
 var sharedGateSession;
 var PASSTHROUGH_SOURCE_TYPES = [
   "bigquery",
@@ -289909,7 +290557,7 @@ function passthroughSourceType(sourceConnection) {
   throw new BadRequestError(`Cannot materialize a '${type}' source into a storage destination: the ` + `native query-passthrough build supports source connections of type ` + `${PASSTHROUGH_SOURCE_TYPES.join(", ")} only.`);
 }
 function createIsolatedBuildSession(sessionName) {
-  const workDir = mkdtempSync(path11.join(os3.tmpdir(), "malloy-build-"));
+  const workDir = mkdtempSync(path12.join(os4.tmpdir(), "malloy-build-"));
   let session;
   try {
     session = new DuckDBConnection4(sessionName, ":memory:", workDir);
@@ -290065,7 +290713,7 @@ async function attachDestinationReadWrite(session, destinationName, destinationC
     await attachDuckLakeReadWrite(session, destinationName, cfg);
     return;
   }
-  const dbPath = path11.join(environmentPath, `${destinationName}.duckdb`);
+  const dbPath = path12.join(environmentPath, `${destinationName}.duckdb`);
   await session.runSQL(`ATTACH '${escapeSQL(dbPath)}' AS ${quoteIdentifier(destinationName, "duckdb")}`);
 }
 function assertSupportedDestination(destinationName, destinationConnection) {
@@ -290127,6 +290775,21 @@ async function resolveEnvironmentId(repository, environmentName) {
 }
 
 // src/service/materialization_service.ts
+function connectionMetadataLayers(environment, connectionName) {
+  try {
+    const connection = environment.getApiConnection(connectionName);
+    return {
+      default: connection?.queryMetadata ?? null,
+      enforced: connection?.queryMetadataEnforced ?? null
+    };
+  } catch (error) {
+    logger.debug("No query-metadata layers for connection", {
+      connectionName,
+      error
+    });
+    return { default: null, enforced: null };
+  }
+}
 var STAGING_ID_LEN = 12;
 function stagingSuffix(sourceEntityId) {
   return `_${sourceEntityId.replace(/-/g, "").substring(0, STAGING_ID_LEN)}`;
@@ -290283,7 +290946,8 @@ class MaterializationService {
       buildInstructions,
       referenceManifest: options.referenceManifest,
       strictUpstreams: options.strictUpstreams,
-      trigger
+      trigger,
+      runContext: options.runContext ?? undefined
     }, signal));
     return created;
   }
@@ -290324,7 +290988,16 @@ class MaterializationService {
         const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
         ({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
       }
-      const entries = await this.executeInstructedBuild(compiled, environment, instructions, carried, signal, opts.strictUpstreams ?? false, orchestrated ? { environmentId, packageName } : undefined);
+      const entries = await this.executeInstructedBuild(compiled, environment, instructions, carried, signal, opts.strictUpstreams ?? false, orchestrated ? { environmentId, packageName } : undefined, {
+        packageMaterialization: pkg.getMaterializationConfig?.() ?? null,
+        context: {
+          queryClass: "materialize",
+          environment: environmentName,
+          package: packageName,
+          trigger: opts.runContext?.trigger ?? opts.trigger?.toLowerCase(),
+          runId: opts.runContext?.runId ?? id
+        }
+      });
       const sourcesBuilt = instructions.length;
       const sourcesReused = Object.keys(carried).length;
       const durationMs = Date.now() - startedAt;
@@ -290537,7 +291210,7 @@ class MaterializationService {
     }
     return quoteManifestTablePath(physicalTableName, connection.dialectName);
   }
-  async executeInstructedBuild(compiled, environment, instructions, seedEntries, signal, strict = false, owner) {
+  async executeInstructedBuild(compiled, environment, instructions, seedEntries, signal, strict = false, owner, buildMetadata) {
     const { graphs, sources, connectionDigests, connections } = compiled;
     const bySourceID = new Map;
     const bySourceEntityId = new Map;
@@ -290579,7 +291252,7 @@ class MaterializationService {
           if (!orchestratedInstruction && instruction.destination && getPersistStorageMode() !== "off") {
             assertMaterializationEligible(persistSource);
           }
-          const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries);
+          const entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata);
           entries[sourceEntityId] = entry;
           if (entry.storageConnectionName)
             builtThisRun.push(entry);
@@ -290646,7 +291319,39 @@ class MaterializationService {
       });
     }
   }
-  async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries) {
+  buildRunSQLOptions(persistSource, environment, buildMetadata) {
+    if (!buildMetadata)
+      return {};
+    const connectionLayers = connectionMetadataLayers(environment, persistSource.connectionName);
+    const resolved = mergeQueryMetadata({
+      connection: connectionLayers.default,
+      enforced: connectionLayers.enforced,
+      model: resolveQueryMetadata(persistSource, buildMetadata.packageMaterialization),
+      context: { ...buildMetadata.context, source: persistSource.name }
+    });
+    if (resolved.drops.length > 0) {
+      logger.warn("Dropped query-metadata properties for a build", {
+        sourceName: persistSource.name,
+        drops: resolved.drops
+      });
+    }
+    return resolved.metadata ? { queryMetadata: resolved.metadata } : {};
+  }
+  dropRunSQLOptions(environment, connectionName, environmentName, packageName, materializationId) {
+    const connectionLayers = connectionMetadataLayers(environment, connectionName);
+    const resolved = mergeQueryMetadata({
+      connection: connectionLayers.default,
+      enforced: connectionLayers.enforced,
+      context: {
+        queryClass: "ops",
+        environment: environmentName,
+        package: packageName,
+        runId: materializationId
+      }
+    });
+    return resolved.metadata ? { queryMetadata: resolved.metadata } : {};
+  }
+  async buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, builtEntries, buildMetadata) {
     const sourceEntityId = instruction.sourceEntityId;
     const physicalTableName = instruction.physicalTableName;
     const isStorageBuild = !!instruction.destination && getPersistStorageMode() !== "off";
@@ -290663,6 +291368,7 @@ class MaterializationService {
       const publicBuildSQL = projectToPublicColumns(persistSource, buildSQL);
       return this.buildOneSourceIntoStorage(persistSource, instruction, manifest, environment, publicBuildSQL, builtEntries, dependsOnStorageUpstream);
     }
+    const runOptions = this.buildRunSQLOptions(persistSource, environment, buildMetadata);
     const bareName = bareTableName(physicalTableName);
     const stagingTableName = `${physicalTableName}${stagingSuffix(sourceEntityId)}`;
     const dialect = persistSource.dialectName;
@@ -290670,14 +291376,14 @@ class MaterializationService {
     const quotedPhysical = quoteTablePath(physicalTableName, dialect);
     const quotedBareName = quoteIdentifier(bareName, dialect);
     const startTime = performance.now();
-    await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
+    await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
     try {
-      await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`);
-      await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`);
-      await connection.runSQL(`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`);
+      await connection.runSQL(`CREATE TABLE ${quotedStaging} AS (${buildSQL})`, runOptions);
+      await connection.runSQL(`DROP TABLE IF EXISTS ${quotedPhysical}`, runOptions);
+      await connection.runSQL(`ALTER TABLE ${quotedStaging} RENAME TO ${quotedBareName}`, runOptions);
     } catch (err) {
       try {
-        await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quotedStaging}`, runOptions);
       } catch (cleanupErr) {
         logger.warn("Failed to clean up staging table after a failed build; physical leak", {
           stagingTableName,
@@ -290970,8 +291676,9 @@ class MaterializationService {
           connectionCache.set(connectionName, connection);
         }
         const dialect = connection.dialectName;
-        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(physicalTableName, dialect)}`);
-        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(`${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`, dialect)}`);
+        const dropOptions = this.dropRunSQLOptions(environment, connectionName, environmentName, packageName, m.id);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(physicalTableName, dialect)}`, dropOptions);
+        await connection.runSQL(`DROP TABLE IF EXISTS ${quoteTablePath(`${physicalTableName}${stagingSuffix(entry.sourceEntityId)}`, dialect)}`, dropOptions);
         recordDropTables("success", "in_warehouse");
         logger.info("Dropped materialized table on delete", {
           materializationId: m.id,
@@ -291286,6 +291993,8 @@ function parseArgs() {
       i++;
     } else if (arg === "--init") {
       process.env.INITIALIZE_STORAGE = "true";
+    } else if (arg === "--no-mcp-config") {
+      process.env.PUBLISHER_NO_MCP_CONFIG = "true";
     } else if (arg === "--watch-env" && args[i + 1]) {
       const existing = process.env.PUBLISHER_WATCH || "";
       process.env.PUBLISHER_WATCH = existing ? `${existing},${args[i + 1]}` : args[i + 1];
@@ -291304,6 +292013,7 @@ function parseArgs() {
       console.log("  --shutdown_drain_duration_seconds <number>  Time in seconds to keep service in draining state before closing servers (default: 0)");
       console.log("  --shutdown_graceful_close_timeout_seconds <number>  Time in seconds to wait after closing servers before exit (default: 0)");
       console.log("  --init                 Wipe persisted storage and re-sync it from the config (default: false)");
+      console.log("  --no-mcp-config        Do not write .mcp.json into the working directory (default: it is written, so an agent opened here finds this server; skipped when the directory already has one, is your home directory or the filesystem root, is inside a git working tree, or the MCP port bound is not the one requested)");
       console.log("  --watch-env <name>     Enable dev-mode watch for the named environment.");
       console.log("                         Mounts local-dir packages in-place (symlink, not");
       console.log("                         copy) so source-edit live reload works. A comma-");
@@ -291320,15 +292030,17 @@ function parseArgs() {
 parseArgs();
 getPersistStorageMode();
 getPersistCollisionEnforce();
+getQueryMetadataMode();
 var PUBLISHER_PORT = Number(process.env.PUBLISHER_PORT || 4000);
 var PUBLISHER_HOST = process.env.PUBLISHER_HOST || "0.0.0.0";
 var MCP_PORT = Number(process.env.MCP_PORT || 4040);
+var MCP_CONFIG_ENABLED = mcpConfigEnabled();
 var MCP_ENDPOINT = "/mcp";
 var SHUTDOWN_DRAIN_DURATION_SECONDS = Number(process.env.SHUTDOWN_DRAIN_DURATION_SECONDS || 0);
 var SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS = Number(process.env.SHUTDOWN_GRACEFUL_CLOSE_TIMEOUT_SECONDS || 0);
 var __filename_esm = fileURLToPath7(import.meta.url);
-var ROOT = path12.join(path12.dirname(__filename_esm), "app");
-var SERVER_ROOT = path12.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
+var ROOT = path13.join(path13.dirname(__filename_esm), "app");
+var SERVER_ROOT = path13.resolve(process.cwd(), process.env.SERVER_ROOT || ".");
 var API_PREFIX2 = "/api/v0";
 var isDevelopment = process.env["NODE_ENV"] === "development";
 var app = import_express.default();
@@ -291426,7 +292138,7 @@ mcpApp.all(MCP_ENDPOINT, async (req, res) => {
     }
   }
 });
-var PUBLISHER_RUNTIME_PATH = path12.join(path12.dirname(__filename_esm), "runtime", "publisher.js");
+var PUBLISHER_RUNTIME_PATH = path13.join(path13.dirname(__filename_esm), "runtime", "publisher.js");
 app.get("/sdk/publisher.js", (_req, res) => {
   res.type("application/javascript");
   res.setHeader("cache-control", "public, max-age=60");
@@ -291444,7 +292156,7 @@ async function serveFromPackage(req, res) {
   try {
     const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
     const pkg = await environment.getPackage(req.params.packageName, false);
-    const publicRoot = path12.join(pkg.getPackagePath(), "public");
+    const publicRoot = path13.join(pkg.getPackagePath(), "public");
     let subPath = subPathRaw;
     if (subPath === "" || subPath.endsWith("/")) {
       subPath = subPath + "index.html";
@@ -291462,12 +292174,12 @@ async function serveFromPackage(req, res) {
       }
       return;
     }
-    const rel = path12.relative(realPublicRoot, realFullPath);
-    if (rel.startsWith("..") || path12.isAbsolute(rel)) {
+    const rel = path13.relative(realPublicRoot, realFullPath);
+    if (rel.startsWith("..") || path13.isAbsolute(rel)) {
       res.status(403).end();
       return;
     }
-    const ext = path12.extname(realFullPath).toLowerCase();
+    const ext = path13.extname(realFullPath).toLowerCase();
     if (ext === ".html" || ext === ".htm") {
       const frameAncestors = process.env.PUBLISHER_FRAME_ANCESTORS || "*";
       res.setHeader("Content-Security-Policy", `frame-ancestors ${frameAncestors}`);
@@ -291517,11 +292229,11 @@ function stripNonTagText(input) {
   return current;
 }
 async function listPackagePages(environmentName, packageName, publicRoot) {
-  const fs9 = await import("fs/promises");
+  const fs10 = await import("fs/promises");
   const out = [];
   let realPublicRoot;
   try {
-    realPublicRoot = await fs9.realpath(publicRoot);
+    realPublicRoot = await fs10.realpath(publicRoot);
   } catch {
     return out;
   }
@@ -291530,31 +292242,31 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
       return;
     let entries;
     try {
-      entries = await fs9.readdir(dir, { withFileTypes: true });
+      entries = await fs10.readdir(dir, { withFileTypes: true });
     } catch {
       return;
     }
     for (const entry of entries) {
       if (entry.name.startsWith(".") || entry.name === "node_modules")
         continue;
-      const full = path12.join(dir, entry.name);
+      const full = path13.join(dir, entry.name);
       let realFull;
       try {
-        realFull = await fs9.realpath(full);
+        realFull = await fs10.realpath(full);
       } catch {
         continue;
       }
-      const contained = path12.relative(realPublicRoot, realFull);
-      if (contained.startsWith("..") || path12.isAbsolute(contained))
+      const contained = path13.relative(realPublicRoot, realFull);
+      if (contained.startsWith("..") || path13.isAbsolute(contained))
         continue;
       if (entry.isDirectory()) {
         await walk(full, depth + 1);
       } else if (entry.isFile() && (entry.name.endsWith(".html") || entry.name.endsWith(".htm"))) {
-        const rel = path12.relative(publicRoot, full).replace(/\\/g, "/");
+        const rel = path13.relative(publicRoot, full).replace(/\\/g, "/");
         let title = rel;
         let fit;
         try {
-          const fh = await fs9.open(full, "r");
+          const fh = await fs10.open(full, "r");
           try {
             const buf = Buffer.alloc(4096);
             const { bytesRead } = await fh.read(buf, 0, 4096, 0);
@@ -291595,14 +292307,14 @@ async function listPackagePages(environmentName, packageName, publicRoot) {
 }
 if (!isDevelopment) {
   app.use("/", import_express.default.static(ROOT));
-  app.use("/api-doc.html", import_express.default.static(path12.join(ROOT, "api-doc.html")));
+  app.use("/api-doc.html", import_express.default.static(path13.join(ROOT, "api-doc.html")));
 } else {
   app.use(`${API_PREFIX2}`, loggerMiddleware);
   app.use(import_http_proxy_middleware.createProxyMiddleware({
     target: "http://localhost:5173",
     changeOrigin: true,
     ws: true,
-    pathFilter: (path13) => !path13.startsWith("/api/") && !path13.startsWith("/metrics") && !path13.startsWith("/health")
+    pathFilter: (path14) => !path14.startsWith("/api/") && !path14.startsWith("/metrics") && !path14.startsWith("/health")
   }));
 }
 var setVersionIdError2 = (res) => {
@@ -291627,7 +292339,7 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/page
   try {
     const environment = await environmentStore.getEnvironment(req.params.environmentName, false);
     const pkg = await environment.getPackage(req.params.packageName, false);
-    const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path12.join(pkg.getPackagePath(), "public"));
+    const pages = await listPackagePages(req.params.environmentName, req.params.packageName, path13.join(pkg.getPackagePath(), "public"));
     res.json(pages);
   } catch (error) {
     logger.error("Failed to list package pages", { error });
@@ -291924,7 +292636,10 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/con
 });
 app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
   try {
-    res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options));
+    res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, undefined, {
+      queryMetadata: req.body?.queryMetadata,
+      queryClass: req.body?.queryClass
+    }));
   } catch (error) {
     logger.error(error);
     const { json, status } = internalErrorToHttpError(error);
@@ -291933,7 +292648,10 @@ app.post(`${API_PREFIX2}/environments/:environmentName/connections/:connectionNa
 });
 app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/connections/:connectionName/sqlQuery`, queryConcurrency(), async (req, res) => {
   try {
-    res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName));
+    res.status(200).json(await connectionController.getConnectionQueryData(req.params.environmentName, req.params.connectionName, req.body.sqlStatement, req.body?.options, req.params.packageName, {
+      queryMetadata: req.body?.queryMetadata,
+      queryClass: req.body?.queryClass
+    }));
   } catch (error) {
     logger.error(error);
     const { json, status } = internalErrorToHttpError(error);
@@ -292133,7 +292851,11 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mod
   }
   try {
     const modelPath = req.params["0"];
-    const result = await queryController.getQuery(req.params.environmentName, req.params.packageName, modelPath, req.body.sourceName, req.body.queryName, req.body.query, req.body.compactJson === true, req.body.filterParams ?? req.body.sourceFilters, req.body.bypassFilters === true ? true : undefined, req.body.givens);
+    const result = await queryController.getQuery(req.params.environmentName, req.params.packageName, modelPath, req.body.sourceName, req.body.queryName, req.body.query, req.body.compactJson === true, req.body.filterParams ?? req.body.sourceFilters, req.body.bypassFilters === true ? true : undefined, req.body.givens, {
+      queryMetadata: req.body?.queryMetadata,
+      queryClass: req.body?.queryClass,
+      versionId: req.body?.versionId
+    });
     setFilterDeprecationHeaders(res, {
       filterParams: req.body.filterParams ?? req.body.sourceFilters,
       bypassFilters: req.body.bypassFilters === true ? true : undefined
@@ -292231,7 +292953,7 @@ registerLegacyRoutes(app, {
   materializationController
 });
 if (!isDevelopment) {
-  const SPA_INDEX = path12.resolve(ROOT, "index.html");
+  const SPA_INDEX = path13.resolve(ROOT, "index.html");
   app.get("*", (req, res) => {
     res.sendFile(SPA_INDEX, (err) => {
       if (!err)
@@ -292284,8 +293006,24 @@ mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
     }
   }
 });
-var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, () => {
-  logger.info(`MCP server listening at http://${PUBLISHER_HOST}:${MCP_PORT}`);
+var mcpServer = mcpApp.listen(MCP_PORT, PUBLISHER_HOST, function() {
+  const boundPort = resolveBoundPort(this.address(), MCP_PORT);
+  const bound = this.address();
+  const boundHost = typeof bound === "object" && bound ? bound.address : PUBLISHER_HOST;
+  logger.info(`MCP server listening at http://${boundHost.includes(":") ? `[${boundHost}]` : boundHost}:${boundPort}`);
+  if (MCP_CONFIG_ENABLED) {
+    try {
+      const endpoint = mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort);
+      logMcpConfigOutcome(ensureMcpConfig({
+        dir: process.cwd(),
+        endpoint,
+        requestedPort: MCP_PORT,
+        boundPort
+      }));
+    } catch (error) {
+      logger.info(`Could not set up ${MCP_CONFIG_FILENAME} (${error instanceof Error ? error.message : String(error)}). To connect an agent, run: ${addCommand(mcpEndpoint(resolveClientHost(this.address(), PUBLISHER_HOST), boundPort))}`);
+    }
+  }
 });
 mcpServer.timeout = 600000;
 mcpServer.keepAliveTimeout = 600000;