@malloy-publisher/server 0.0.237 → 0.0.238

package/dist/server.mjs

@@ -156044,6 +156044,8 @@ function internalErrorToHttpError(error) {
     return httpError(413, error.message);
   } else if (error instanceof QueryTimeoutError) {
     return httpError(504, error.message);
+  } else if (error instanceof NotImplementedError) {
+    return httpError(501, error.message);
   } else {
     return httpError(500, error.message);
   }
@@ -156057,7 +156059,7 @@ function httpError(code, message) {
     }
   };
 }
-var NotImplementedError, BadRequestError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, MaterializationEligibilityError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
+var NotImplementedError, BadRequestError, InvalidArgumentError, EnvironmentNotFoundError, PackageNotFoundError, ModelNotFoundError, ConnectionNotFoundError, ConnectionError, DestinationNotFoundError, ConnectionAuthError, UnsupportedCatalogFormatError, ModelCompilationError, MaterializationEligibilityError, FrozenConfigError, AccessDeniedError, NotQueryableError, MaterializationNotFoundError, MaterializationConflictError, InvalidStateTransitionError, ServiceUnavailableError, PayloadTooLargeError, ResponseUnserializableError, QueryTimeoutError;
 var init_errors = __esm(() => {
   init_constants();
   NotImplementedError = class NotImplementedError extends Error {
@@ -156070,6 +156072,8 @@ var init_errors = __esm(() => {
       super(message);
     }
   };
+  InvalidArgumentError = class InvalidArgumentError extends BadRequestError {
+  };
   EnvironmentNotFoundError = class EnvironmentNotFoundError extends Error {
     constructor(message) {
       super(message);
@@ -241890,7 +241894,7 @@ var require_luxon = __commonJS((exports) => {
     }
   }
 
-  class InvalidArgumentError extends LuxonError {
+  class InvalidArgumentError2 extends LuxonError {
   }
 
   class ZoneIsAbstractError extends LuxonError {
@@ -243233,10 +243237,10 @@ var require_luxon = __commonJS((exports) => {
     if (settings == null) {
       return null;
     } else if (typeof settings !== "object") {
-      throw new InvalidArgumentError("Week settings must be an object");
+      throw new InvalidArgumentError2("Week settings must be an object");
     } else {
       if (!integerBetween(settings.firstDay, 1, 7) || !integerBetween(settings.minimalDays, 1, 7) || !Array.isArray(settings.weekend) || settings.weekend.some((v) => !integerBetween(v, 1, 7))) {
-        throw new InvalidArgumentError("Invalid week settings");
+        throw new InvalidArgumentError2("Invalid week settings");
       }
       return {
         firstDay: settings.firstDay,
@@ -243367,7 +243371,7 @@ var require_luxon = __commonJS((exports) => {
   function asNumber2(value) {
     const numericValue = Number(value);
     if (typeof value === "boolean" || value === "" || !Number.isFinite(numericValue))
-      throw new InvalidArgumentError(`Invalid unit value ${value}`);
+      throw new InvalidArgumentError2(`Invalid unit value ${value}`);
     return numericValue;
   }
   function normalizeObject(obj, normalizer) {
@@ -244213,7 +244217,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static fromObject(obj, opts = {}) {
       if (obj == null || typeof obj !== "object") {
-        throw new InvalidArgumentError(`Duration.fromObject: argument expected to be an object, got ${obj === null ? "null" : typeof obj}`);
+        throw new InvalidArgumentError2(`Duration.fromObject: argument expected to be an object, got ${obj === null ? "null" : typeof obj}`);
       }
       return new Duration({
         values: normalizeObject(obj, Duration.normalizeUnit),
@@ -244230,7 +244234,7 @@ var require_luxon = __commonJS((exports) => {
       } else if (typeof durationLike === "object") {
         return Duration.fromObject(durationLike);
       } else {
-        throw new InvalidArgumentError(`Unknown duration argument ${durationLike} of type ${typeof durationLike}`);
+        throw new InvalidArgumentError2(`Unknown duration argument ${durationLike} of type ${typeof durationLike}`);
       }
     }
     static fromISO(text, opts) {
@@ -244251,7 +244255,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static invalid(reason, explanation = null) {
       if (!reason) {
-        throw new InvalidArgumentError("need to specify a reason the Duration is invalid");
+        throw new InvalidArgumentError2("need to specify a reason the Duration is invalid");
       }
       const invalid = reason instanceof Invalid ? reason : new Invalid(reason, explanation);
       if (Settings.throwOnInvalid) {
@@ -244616,7 +244620,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static invalid(reason, explanation = null) {
       if (!reason) {
-        throw new InvalidArgumentError("need to specify a reason the Interval is invalid");
+        throw new InvalidArgumentError2("need to specify a reason the Interval is invalid");
       }
       const invalid = reason instanceof Invalid ? reason : new Invalid(reason, explanation);
       if (Settings.throwOnInvalid) {
@@ -245887,7 +245891,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static fromMillis(milliseconds, options = {}) {
       if (!isNumber2(milliseconds)) {
-        throw new InvalidArgumentError(`fromMillis requires a numerical input, but received a ${typeof milliseconds} with value ${milliseconds}`);
+        throw new InvalidArgumentError2(`fromMillis requires a numerical input, but received a ${typeof milliseconds} with value ${milliseconds}`);
       } else if (milliseconds < -MAX_DATE || milliseconds > MAX_DATE) {
         return DateTime.invalid("Timestamp out of range");
       } else {
@@ -245900,7 +245904,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static fromSeconds(seconds, options = {}) {
       if (!isNumber2(seconds)) {
-        throw new InvalidArgumentError("fromSeconds requires a numerical input");
+        throw new InvalidArgumentError2("fromSeconds requires a numerical input");
       } else {
         return new DateTime({
           ts: seconds * 1000,
@@ -245985,7 +245989,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static fromFormat(text, fmt, opts = {}) {
       if (isUndefined2(text) || isUndefined2(fmt)) {
-        throw new InvalidArgumentError("fromFormat requires an input string and a format");
+        throw new InvalidArgumentError2("fromFormat requires an input string and a format");
       }
       const {
         locale = null,
@@ -246010,7 +246014,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static invalid(reason, explanation = null) {
       if (!reason) {
-        throw new InvalidArgumentError("need to specify a reason the DateTime is invalid");
+        throw new InvalidArgumentError2("need to specify a reason the DateTime is invalid");
       }
       const invalid = reason instanceof Invalid ? reason : new Invalid(reason, explanation);
       if (Settings.throwOnInvalid) {
@@ -246570,13 +246574,13 @@ var require_luxon = __commonJS((exports) => {
     }
     static min(...dateTimes) {
       if (!dateTimes.every(DateTime.isDateTime)) {
-        throw new InvalidArgumentError("min requires all arguments be DateTimes");
+        throw new InvalidArgumentError2("min requires all arguments be DateTimes");
       }
       return bestBy(dateTimes, (i) => i.valueOf(), Math.min);
     }
     static max(...dateTimes) {
       if (!dateTimes.every(DateTime.isDateTime)) {
-        throw new InvalidArgumentError("max requires all arguments be DateTimes");
+        throw new InvalidArgumentError2("max requires all arguments be DateTimes");
       }
       return bestBy(dateTimes, (i) => i.valueOf(), Math.max);
     }
@@ -246607,7 +246611,7 @@ var require_luxon = __commonJS((exports) => {
     }
     static fromFormatParser(text, formatParser, opts = {}) {
       if (isUndefined2(text) || isUndefined2(formatParser)) {
-        throw new InvalidArgumentError("fromFormatParser requires an input string and a format parser");
+        throw new InvalidArgumentError2("fromFormatParser requires an input string and a format parser");
       }
       const {
         locale = null,
@@ -246618,7 +246622,7 @@ var require_luxon = __commonJS((exports) => {
         defaultToEN: true
       });
       if (!localeToUse.equals(formatParser.locale)) {
-        throw new InvalidArgumentError(`fromFormatParser called with a locale of ${localeToUse}, ` + `but the format parser was created for ${formatParser.locale}`);
+        throw new InvalidArgumentError2(`fromFormatParser called with a locale of ${localeToUse}, ` + `but the format parser was created for ${formatParser.locale}`);
       }
       const {
         result,
@@ -246707,7 +246711,7 @@ var require_luxon = __commonJS((exports) => {
     } else if (dateTimeish && typeof dateTimeish === "object") {
       return DateTime.fromObject(dateTimeish);
     } else {
-      throw new InvalidArgumentError(`Unknown datetime argument: ${dateTimeish}, of type ${typeof dateTimeish}`);
+      throw new InvalidArgumentError2(`Unknown datetime argument: ${dateTimeish}, of type ${typeof dateTimeish}`);
     }
   }
   var VERSION3 = "3.7.2";
@@ -261369,6 +261373,7 @@ var getEmbeddingConfig = () => {
   }
   return { apiKey, model, baseUrl, dimensions };
 };
+var schemaEmbeddingEnabled = () => parseBoolEnv("EMBEDDING_INDEX_CONNECTION_SCHEMA") ?? false;
 var DEFAULT_SCHEDULER_INTERVAL_MS = 60000;
 var MIN_SCHEDULER_INTERVAL_MS = 1000;
 var DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK = 10;
@@ -261763,7 +261768,7 @@ function ensureCapTelemetry() {
   const meter2 = publisherMeter();
   if (!capExceededCounter) {
     capExceededCounter = meter2.createCounter("publisher_query_cap_exceeded_total", {
-      description: "Queries rejected with 413 because the row or byte cap was exceeded. Labels: cap_type ('rows'|'bytes'), source ('connection_sql'|'model_query'|'notebook_cell')."
+      description: "413s for an oversized response. cap_type: rows|bytes exceeded that cap; unserializable could not be turned into JSON at all, so no cap need have been exceeded. source: connection_sql|model_query|notebook_cell."
     });
   }
   if (!configGaugesInstalled) {
@@ -267478,6 +267483,7 @@ class ConnectionService {
 }
 
 // src/service/db_utils.ts
+init_errors();
 init_logger();
 var import_bigquery = __toESM(require_src121(), 1);
 import { ClientSecretCredential } from "@azure/identity";
@@ -267486,6 +267492,50 @@ import { ContainerClient } from "@azure/storage-blob";
 // src/service/gcs_s3_utils.ts
 init_logger();
 var import_client_s3 = __toESM(require_dist_cjs75(), 1);
+
+// src/service/introspection_sql.ts
+init_logger();
+var BACKSLASH_ESCAPE_DIALECTS = new Set(["databricks", "mysql", "snowflake"]);
+var UNSUPPORTED_LITERAL_DIALECTS = new Map([
+  [
+    "bigquery",
+    "GoogleSQL does not accept '' as an escaped quote, so this function cannot build a correct BigQuery literal. BigQuery introspection goes through the @google-cloud/bigquery client instead of building SQL."
+  ]
+]);
+var DOUBLED_ESCAPE_DIALECTS = new Set([
+  "ducklake",
+  "duckdb",
+  "motherduck",
+  "postgres",
+  "publisher",
+  "trino"
+]);
+function sqlLiteral(value, connectionType) {
+  const dialect = (connectionType ?? "").toLowerCase();
+  const unsupported = UNSUPPORTED_LITERAL_DIALECTS.get(dialect);
+  if (unsupported) {
+    throw new Error(`Cannot build a SQL literal for "${dialect}". ${unsupported}`);
+  }
+  const backslash = BACKSLASH_ESCAPE_DIALECTS.has(dialect);
+  if (dialect && !backslash && !DOUBLED_ESCAPE_DIALECTS.has(dialect)) {
+    throw new Error(`Unclassified SQL dialect "${connectionType}": add it to BACKSLASH_ESCAPE_DIALECTS or DOUBLED_ESCAPE_DIALECTS in introspection_sql.ts before building SQL for it, or to UNSUPPORTED_LITERAL_DIALECTS if, like BigQuery, it does not accept '' as an escaped quote.`);
+  }
+  const escaped = backslash ? value.replace(/\\/g, "\\\\") : value;
+  return escaped.replace(/'/g, "''");
+}
+var INTROSPECTION_ROW_LIMIT = 1e5;
+async function runIntrospectionSQL(malloyConnection, sql) {
+  const result = await malloyConnection.runSQL(sql, {
+    rowLimit: INTROSPECTION_ROW_LIMIT
+  });
+  const rowCount = Array.isArray(result) ? result.length : result?.rows?.length ?? 0;
+  if (rowCount === INTROSPECTION_ROW_LIMIT) {
+    logger.warn("Schema introspection hit the row cap; the result may be truncated", { rowLimit: INTROSPECTION_ROW_LIMIT });
+  }
+  return result;
+}
+
+// src/service/gcs_s3_utils.ts
 function gcsConnectionToCredentials(gcsConnection) {
   return {
     type: "gcs",
@@ -267604,22 +267654,22 @@ async function getTableSchema(malloyConnection, credentials, bucketName, fileKey
     let describeQuery;
     switch (fileType) {
       case "csv":
-        describeQuery = `DESCRIBE SELECT * FROM read_csv('${uri}', auto_detect=true) LIMIT 1`;
+        describeQuery = `DESCRIBE SELECT * FROM read_csv('${sqlLiteral(uri, "duckdb")}', auto_detect=true) LIMIT 1`;
         break;
       case "parquet":
-        describeQuery = `DESCRIBE SELECT * FROM read_parquet('${uri}') LIMIT 1`;
+        describeQuery = `DESCRIBE SELECT * FROM read_parquet('${sqlLiteral(uri, "duckdb")}') LIMIT 1`;
         break;
       case "json":
-        describeQuery = `DESCRIBE SELECT * FROM read_json('${uri}', auto_detect=true) LIMIT 1`;
+        describeQuery = `DESCRIBE SELECT * FROM read_json('${sqlLiteral(uri, "duckdb")}', auto_detect=true) LIMIT 1`;
         break;
       case "jsonl":
-        describeQuery = `DESCRIBE SELECT * FROM read_json('${uri}', format='newline_delimited', auto_detect=true) LIMIT 1`;
+        describeQuery = `DESCRIBE SELECT * FROM read_json('${sqlLiteral(uri, "duckdb")}', format='newline_delimited', auto_detect=true) LIMIT 1`;
         break;
       default:
         logger.warn(`Unsupported file type for ${fileKey}`);
         return { resource: uri, columns: [] };
     }
-    const result = await malloyConnection.runSQL(describeQuery);
+    const result = await runIntrospectionSQL(malloyConnection, describeQuery);
     const rows = standardizeRunSQLResult(result);
     const columns = rows.map((row) => {
       const typedRow = row;
@@ -267727,12 +267777,19 @@ async function listCloudDirectorySchemas(credentials) {
 }
 
 // src/service/db_utils.ts
-function sqlInFilter(columnName, values) {
+function sqlInFilter(columnName, values, connectionType) {
   if (!values || values.length === 0)
     return "";
-  const escaped = values.map((v) => `'${v.replace(/'/g, "''")}'`);
+  const escaped = values.map((v) => `'${sqlLiteral(v, connectionType)}'`);
   return `AND ${columnName} IN (${escaped.join(", ")})`;
 }
+var SAFE_SQL_IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_$]*$/;
+function assertSafeSqlIdentifier(value, what) {
+  if (!SAFE_SQL_IDENTIFIER.test(value)) {
+    throw new InvalidArgumentError(`Invalid ${what} "${value}": expected a plain identifier (letters, digits, underscore or dollar, not starting with a digit).`);
+  }
+  return value;
+}
 function groupColumnRowsIntoTables(rows, buildResource) {
   const tableMap = new Map;
   for (const row of rows) {
@@ -267820,7 +267877,7 @@ async function getSchemasForPostgres(connection, malloyConnection) {
     throw new Error("Postgres connection is required");
   }
   try {
-    const result = await malloyConnection.runSQL("SELECT row_to_json(t) as row FROM (SELECT schema_name FROM information_schema.schemata ORDER BY schema_name) t");
+    const result = await runIntrospectionSQL(malloyConnection, "SELECT row_to_json(t) as row FROM (SELECT schema_name FROM information_schema.schemata ORDER BY schema_name) t");
     const rows = standardizeRunSQLResult2(result);
     return rows.map((row) => {
       const typedRow = row;
@@ -267857,13 +267914,13 @@ async function getSchemasForSnowflake(connection, malloyConnection) {
     const schema = connection.snowflakeConnection.schema;
     const filters = [];
     if (database) {
-      filters.push(`CATALOG_NAME = '${database}'`);
+      filters.push(`CATALOG_NAME = '${sqlLiteral(database, connection.type)}'`);
     }
     if (schema) {
-      filters.push(`SCHEMA_NAME = '${schema}'`);
+      filters.push(`SCHEMA_NAME = '${sqlLiteral(schema, connection.type)}'`);
     }
     const whereClause = filters.length > 0 ? `WHERE ${filters.join(" AND ")}` : "";
-    const result = await malloyConnection.runSQL(`SELECT CATALOG_NAME, SCHEMA_NAME, SCHEMA_OWNER FROM ${database ? `${database}.` : ""}INFORMATION_SCHEMA.SCHEMATA ${whereClause} ORDER BY SCHEMA_NAME`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT CATALOG_NAME, SCHEMA_NAME, SCHEMA_OWNER FROM ${database ? `${database}.` : ""}INFORMATION_SCHEMA.SCHEMATA ${whereClause} ORDER BY SCHEMA_NAME`);
     const rows = standardizeRunSQLResult2(result);
     return rows.map((row) => {
       const typedRow = row;
@@ -267890,7 +267947,7 @@ async function getSchemasForTrino(connection, malloyConnection) {
     let allRows = [];
     if (connection.trinoConnection.catalog) {
       const catalog = connection.trinoConnection.catalog;
-      const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+      const result = await runIntrospectionSQL(malloyConnection, `SELECT schema_name FROM ${assertSafeSqlIdentifier(catalog, "catalog name")}.information_schema.schemata ORDER BY schema_name`);
       const rows = standardizeRunSQLResult2(result);
       allRows = rows.map((row) => {
         const r = row;
@@ -267900,14 +267957,14 @@ async function getSchemasForTrino(connection, malloyConnection) {
         };
       });
     } else {
-      const catalogsResult = await malloyConnection.runSQL(`SHOW CATALOGS`);
+      const catalogsResult = await runIntrospectionSQL(malloyConnection, `SHOW CATALOGS`);
       const catalogNames = standardizeRunSQLResult2(catalogsResult).map((row) => {
         const r = row;
         return String(r.Catalog ?? r.catalog ?? "");
       });
       for (const catalog of catalogNames) {
         try {
-          const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+          const result = await runIntrospectionSQL(malloyConnection, `SELECT schema_name FROM ${assertSafeSqlIdentifier(catalog, "catalog name")}.information_schema.schemata ORDER BY schema_name`);
           const rows = standardizeRunSQLResult2(result);
           for (const row of rows) {
             const r = row;
@@ -267930,6 +267987,8 @@ async function getSchemasForTrino(connection, malloyConnection) {
       };
     });
   } catch (error) {
+    if (error instanceof BadRequestError)
+      throw error;
     logger.error(`Error getting schemas for Trino connection ${connection.name}`, { error });
     throw new Error(`Failed to get schemas for Trino connection ${connection.name}: ${error.message}`);
   }
@@ -267943,7 +268002,7 @@ async function getSchemasForDatabricks(connection, malloyConnection) {
     let allRows = [];
     if (connection.databricksConnection.defaultCatalog) {
       const catalog = connection.databricksConnection.defaultCatalog;
-      const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+      const result = await runIntrospectionSQL(malloyConnection, `SELECT schema_name FROM ${assertSafeSqlIdentifier(catalog, "catalog name")}.information_schema.schemata ORDER BY schema_name`);
       const rows = standardizeRunSQLResult2(result);
       allRows = rows.map((row) => {
         const r = row;
@@ -267953,14 +268012,14 @@ async function getSchemasForDatabricks(connection, malloyConnection) {
         };
       });
     } else {
-      const catalogsResult = await malloyConnection.runSQL(`SHOW CATALOGS`);
+      const catalogsResult = await runIntrospectionSQL(malloyConnection, `SHOW CATALOGS`);
       const catalogNames = standardizeRunSQLResult2(catalogsResult).map((row) => {
         const r = row;
         return String(r.catalog ?? r.Catalog ?? r.catalog_name ?? "");
       });
       for (const catalog of catalogNames) {
         try {
-          const result = await malloyConnection.runSQL(`SELECT schema_name FROM ${catalog}.information_schema.schemata ORDER BY schema_name`);
+          const result = await runIntrospectionSQL(malloyConnection, `SELECT schema_name FROM ${assertSafeSqlIdentifier(catalog, "catalog name")}.information_schema.schemata ORDER BY schema_name`);
           const rows = standardizeRunSQLResult2(result);
           for (const row of rows) {
             const r = row;
@@ -267984,6 +268043,8 @@ async function getSchemasForDatabricks(connection, malloyConnection) {
       };
     });
   } catch (error) {
+    if (error instanceof BadRequestError)
+      throw error;
     logger.error(`Error getting schemas for Databricks connection ${connection.name}`, { error });
     throw new Error(`Failed to get schemas for Databricks connection ${connection.name}: ${error.message}`);
   }
@@ -267993,7 +268054,7 @@ async function getSchemasForDuckDB(connection, malloyConnection) {
     throw new Error("DuckDB connection is required");
   }
   try {
-    const result = await malloyConnection.runSQL("SELECT DISTINCT schema_name,catalog_name FROM information_schema.schemata ORDER BY catalog_name,schema_name", { rowLimit: 1000 });
+    const result = await runIntrospectionSQL(malloyConnection, "SELECT DISTINCT schema_name,catalog_name FROM information_schema.schemata ORDER BY catalog_name,schema_name");
     const rows = standardizeRunSQLResult2(result);
     const schemas = rows.map((row) => {
       const typedRow = row;
@@ -268049,8 +268110,8 @@ async function getSchemasForMotherDuck(connection, malloyConnection) {
   }
   try {
     const database = connection.motherduckConnection.database;
-    const whereClause = database ? `WHERE catalog_name = '${database}'` : "";
-    const result = await malloyConnection.runSQL(`SELECT DISTINCT schema_name FROM information_schema.schemata ${whereClause} ORDER BY schema_name`);
+    const whereClause = database ? `WHERE catalog_name = '${sqlLiteral(database, connection.type)}'` : "";
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT DISTINCT schema_name FROM information_schema.schemata ${whereClause} ORDER BY schema_name`);
     const rows = standardizeRunSQLResult2(result);
     return rows.map((row) => {
       const typedRow = row;
@@ -268068,8 +268129,8 @@ async function getSchemasForMotherDuck(connection, malloyConnection) {
 }
 async function getSchemasForDuckLake(connection, malloyConnection) {
   try {
-    const catalogName = connection.name;
-    const result = await malloyConnection.runSQL(`SELECT schema_name FROM information_schema.schemata WHERE catalog_name = '${catalogName}' ORDER BY schema_name`, { rowLimit: 1000 });
+    const catalogName = connection.name ?? "";
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT schema_name FROM information_schema.schemata WHERE catalog_name = '${sqlLiteral(catalogName, connection.type)}' ORDER BY schema_name`);
     const rows = standardizeRunSQLResult2(result);
     return rows.map((row) => {
       const typedRow = row;
@@ -268260,28 +268321,29 @@ function isDataFile2(key) {
   const lowerKey = key.toLowerCase();
   return lowerKey.endsWith(".csv") || lowerKey.endsWith(".parquet") || lowerKey.endsWith(".json") || lowerKey.endsWith(".jsonl") || lowerKey.endsWith(".ndjson");
 }
+var DUCKDB_DIALECT = "duckdb";
 async function describeRemoteFile(malloyConnection, fileUri) {
   const pathWithoutQuery = fileUri.split("?")[0];
   const fileType = getFileType2(pathWithoutQuery);
   let describeQuery;
   switch (fileType) {
     case "csv":
-      describeQuery = `DESCRIBE SELECT * FROM read_csv('${fileUri}', auto_detect=true) LIMIT 1`;
+      describeQuery = `DESCRIBE SELECT * FROM read_csv('${sqlLiteral(fileUri, DUCKDB_DIALECT)}', auto_detect=true) LIMIT 1`;
       break;
     case "parquet":
-      describeQuery = `DESCRIBE SELECT * FROM read_parquet('${fileUri}') LIMIT 1`;
+      describeQuery = `DESCRIBE SELECT * FROM read_parquet('${sqlLiteral(fileUri, DUCKDB_DIALECT)}') LIMIT 1`;
       break;
     case "json":
-      describeQuery = `DESCRIBE SELECT * FROM read_json('${fileUri}', auto_detect=true) LIMIT 1`;
+      describeQuery = `DESCRIBE SELECT * FROM read_json('${sqlLiteral(fileUri, DUCKDB_DIALECT)}', auto_detect=true) LIMIT 1`;
       break;
     case "jsonl":
-      describeQuery = `DESCRIBE SELECT * FROM read_json('${fileUri}', format='newline_delimited', auto_detect=true) LIMIT 1`;
+      describeQuery = `DESCRIBE SELECT * FROM read_json('${sqlLiteral(fileUri, DUCKDB_DIALECT)}', format='newline_delimited', auto_detect=true) LIMIT 1`;
       break;
     default:
       logger.warn(`Unsupported file type for file: ${fileUri}`);
       return { resource: fileUri, columns: [] };
   }
-  const result = await malloyConnection.runSQL(describeQuery);
+  const result = await runIntrospectionSQL(malloyConnection, describeQuery);
   const rows = standardizeRunSQLResult2(result);
   const columns = rows.map((row) => {
     const typedRow = row;
@@ -268394,7 +268456,7 @@ async function listTablesForMySQL(connection, schemaName, malloyConnection, tabl
     throw new Error("Mysql connection is required");
   }
   try {
-    const result = await malloyConnection.runSQL(`SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM information_schema.columns WHERE table_schema = '${schemaName}' ${sqlInFilter("TABLE_NAME", tableNames)} ORDER BY TABLE_NAME, ORDINAL_POSITION`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM information_schema.columns WHERE table_schema = '${sqlLiteral(schemaName, connection.type)}' ${sqlInFilter("TABLE_NAME", tableNames, connection.type)} ORDER BY TABLE_NAME, ORDINAL_POSITION`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);
   } catch (error) {
@@ -268407,7 +268469,7 @@ async function listTablesForPostgres(connection, schemaName, malloyConnection, t
     throw new Error("Postgres connection is required");
   }
   try {
-    const result = await malloyConnection.runSQL(`SELECT row_to_json(t) as row FROM (SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${schemaName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position) t`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT row_to_json(t) as row FROM (SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${sqlLiteral(schemaName, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position) t`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);
   } catch (error) {
@@ -268433,12 +268495,15 @@ async function listTablesForSnowflake(connection, schemaName, malloyConnection,
     if (!databaseName) {
       throw new Error(`Cannot resolve database for schema "${schemaName}": provide DATABASE.SCHEMA or configure a database on the connection`);
     }
+    assertSafeSqlIdentifier(databaseName, "database name");
     const qualifiedSchema = `${databaseName}.${schemaOnly}`;
-    const result = await malloyConnection.runSQL(`SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM ${databaseName}.INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${schemaOnly}' ${sqlInFilter("TABLE_NAME", tableNames)} ORDER BY TABLE_NAME, ORDINAL_POSITION`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM ${databaseName}.INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA = '${sqlLiteral(schemaOnly, connection.type)}' ${sqlInFilter("TABLE_NAME", tableNames, connection.type)} ORDER BY TABLE_NAME, ORDINAL_POSITION`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${qualifiedSchema}.${t}`);
   } catch (error) {
     logger.error(`Error getting tables for Snowflake schema ${schemaName} in connection ${connection.name}`, { error });
+    if (error instanceof BadRequestError)
+      throw error;
     throw new Error(`Failed to get tables for Snowflake schema ${schemaName} in connection ${connection.name}: ${error.message}`);
   }
 }
@@ -268457,7 +268522,7 @@ async function listTablesForTrino(connection, schemaName, malloyConnection, tabl
     } else {
       const dotIdx = schemaName.indexOf(".");
       if (dotIdx > 0) {
-        catalogPrefix = `${schemaName.substring(0, dotIdx)}.`;
+        catalogPrefix = `${assertSafeSqlIdentifier(schemaName.substring(0, dotIdx), "catalog name")}.`;
         schemaOnly = schemaName.substring(dotIdx + 1);
       } else {
         catalogPrefix = "";
@@ -268465,11 +268530,13 @@ async function listTablesForTrino(connection, schemaName, malloyConnection, tabl
       }
       resourcePrefix = schemaName;
     }
-    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${schemaOnly}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${sqlLiteral(schemaOnly, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${resourcePrefix}.${t}`);
   } catch (error) {
     logger.error(`Error getting tables for Trino schema ${schemaName} in connection ${connection.name}`, { error });
+    if (error instanceof BadRequestError)
+      throw error;
     throw new Error(`Failed to get tables for Trino schema ${schemaName} in connection ${connection.name}: ${error.message}`);
   }
 }
@@ -268488,7 +268555,7 @@ async function listTablesForDatabricks(connection, schemaName, malloyConnection,
     } else {
       const dotIdx = schemaName.indexOf(".");
       if (dotIdx > 0) {
-        catalogPrefix = `${schemaName.substring(0, dotIdx)}.`;
+        catalogPrefix = `${assertSafeSqlIdentifier(schemaName.substring(0, dotIdx), "catalog name")}.`;
         schemaOnly = schemaName.substring(dotIdx + 1);
       } else {
         catalogPrefix = "";
@@ -268496,11 +268563,13 @@ async function listTablesForDatabricks(connection, schemaName, malloyConnection,
       }
       resourcePrefix = schemaName;
     }
-    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${schemaOnly}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT table_name, column_name, data_type FROM ${catalogPrefix}information_schema.columns WHERE table_schema = '${sqlLiteral(schemaOnly, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${resourcePrefix}.${t}`);
   } catch (error) {
     logger.error(`Error getting tables for Databricks schema ${schemaName} in connection ${connection.name}`, { error });
+    if (error instanceof BadRequestError)
+      throw error;
     throw new Error(`Failed to get tables for Databricks schema ${schemaName} in connection ${connection.name}: ${error.message}`);
   }
 }
@@ -268536,12 +268605,12 @@ async function listTablesForDuckDB(connection, schemaName, malloyConnection, tab
   }
   const dotIdx = schemaName.indexOf(".");
   if (dotIdx < 0) {
-    throw new Error(`DuckDB schema name must be qualified as "catalog.schema", got "${schemaName}"`);
+    throw new InvalidArgumentError(`DuckDB schema name must be qualified as "catalog.schema", got "${schemaName}". List this connection's schemas and use one of those names verbatim.`);
   }
   const catalogName = schemaName.substring(0, dotIdx);
   const actualSchemaName = schemaName.substring(dotIdx + 1);
   try {
-    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${actualSchemaName}' AND table_catalog = '${catalogName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${sqlLiteral(actualSchemaName, connection.type)}' AND table_catalog = '${sqlLiteral(catalogName, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);
   } catch (error) {
@@ -268554,7 +268623,7 @@ async function listTablesForMotherDuck(connection, schemaName, malloyConnection,
     throw new Error("MotherDuck connection is required");
   }
   try {
-    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${schemaName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${sqlLiteral(schemaName, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);
   } catch (error) {
@@ -268569,7 +268638,7 @@ async function listTablesForDuckLake(connection, schemaName, malloyConnection, t
   const catalogName = schemaName.split(".")[0];
   const actualSchemaName = schemaName.split(".")[1];
   try {
-    const result = await malloyConnection.runSQL(`SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${actualSchemaName}' AND table_catalog = '${catalogName}' ${sqlInFilter("table_name", tableNames)} ORDER BY table_name, ordinal_position`);
+    const result = await runIntrospectionSQL(malloyConnection, `SELECT table_name, column_name, data_type FROM information_schema.columns WHERE table_schema = '${sqlLiteral(actualSchemaName, connection.type)}' AND table_catalog = '${sqlLiteral(catalogName, connection.type)}' ${sqlInFilter("table_name", tableNames, connection.type)} ORDER BY table_name, ordinal_position`);
     const rows = standardizeRunSQLResult2(result);
     return groupColumnRowsIntoTables(rows, (t) => `${schemaName}.${t}`);
   } catch (error) {
@@ -268754,7 +268823,7 @@ class ConnectionController {
         const pkg = await environment.getPackage(onlyPackage);
         return await pkg.getMalloyConnection(connectionName);
       }
-      throw new BadRequestError(`Ambiguous "duckdb" connection lookup: environment "${environmentName}" has multiple packages. ` + `Use /environments/${environmentName}/packages/{packageName}/connections/duckdb/... to disambiguate.`);
+      throw new BadRequestError(`Ambiguous "duckdb" connection lookup: environment "${environmentName}" has multiple packages, ` + `and the "duckdb" sandbox exists once per package. Name one of: ${packages.map((p) => p.name).filter(Boolean).join(", ")}. ` + `Over MCP pass it as the packageName argument; over REST use ` + `/environments/${environmentName}/packages/{packageName}/connections/duckdb/...`);
     } else {
       return await environment.getMalloyConnection(connectionName);
     }
@@ -269240,17 +269309,6 @@ class PackageController {
 var import_render_validator = __toESM(require_dist11(), 1);
 init_constants();
 init_errors();
-
-// src/json_utils.ts
-var MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
-function bigIntReplacer(_key, value) {
-  if (typeof value === "bigint") {
-    return value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT ? value.toString() : Number(value);
-  }
-  return value;
-}
-
-// src/controller/query.controller.ts
 init_logger();
 class QueryController {
   environmentStore;
@@ -269276,7 +269334,6 @@ class QueryController {
       const {
         result,
         serializedResult,
-        compactResult,
         rowLimit,
         rowLimitSource,
         queryCorrelationId
@@ -269298,10 +269355,10 @@ class QueryController {
             return null;
           }
         }
-      }), getQueryTimeoutMs());
+      }, compactJson ? "compact" : "full"), getQueryTimeoutMs());
       const renderLogs = import_render_validator.validateRenderTags(result);
       return {
-        result: compactJson ? JSON.stringify(compactResult, bigIntReplacer) : serializedResult,
+        result: serializedResult,
         resource: `${API_PREFIX}/environments/${environmentName}/packages/${packageName}/models/${modelPath}/query`,
         renderLogs: renderLogs.length > 0 ? renderLogs : undefined,
         queryRowLimit: rowLimit,
@@ -277551,7 +277608,7 @@ async function assertServesInDuckDB(sourceName, binding, connections) {
 init_errors();
 init_constants();
 init_logger();
-import { Annotations as Annotations3 } from "@malloydata/malloy";
+import { Annotations as Annotations2 } from "@malloydata/malloy";
 
 // src/service/materialization_eligibility.ts
 init_errors();
@@ -277696,7 +277753,6 @@ function walkForAuthorize(node, seen, depth) {
 
 // src/service/model.ts
 import {
-  Annotations as Annotations2,
   API,
   FixedConnectionMap,
   InMemoryURLReader as InMemoryURLReader2,
@@ -277846,10 +277902,32 @@ function modelAnnotations(modelDef) {
   }
   return folded ?? {};
 }
+function ownModelNotes(modelDef) {
+  const registry = modelDef.modelAnnotations ?? {};
+  const isSameDocument = (id) => id === modelDef.modelID || id.startsWith("internal://");
+  const seen = new Set;
+  const texts = [];
+  const visit = (id) => {
+    if (seen.has(id) || !isSameDocument(id))
+      return;
+    seen.add(id);
+    const entry = registry[id];
+    if (!entry)
+      return;
+    for (const dep of entry.inheritsFrom)
+      visit(dep);
+    texts.push(...ownLevelNoteTexts(entry.ownNotes));
+  };
+  visit(modelDef.modelID);
+  return texts;
+}
 function annotationTexts(annote) {
   const texts = new Annotations(annote).texts();
   return texts.length > 0 ? texts : undefined;
 }
+function ownLevelNoteTexts(annote) {
+  return [...annote?.blockNotes ?? [], ...annote?.notes ?? []].map((note) => note.text);
+}
 
 // src/service/filter.ts
 var VALID_FILTER_TYPES = new Set([
@@ -278065,19 +278143,24 @@ function resolveModelQueryRowLimit(userLimit, { defaultLimit, maxRows }) {
 function queryRowLimitSource(userLimit) {
   return userLimit && userLimit > 0 ? "query" : "server_default";
 }
-function assertWithinModelResponseLimits(rowCount, serializedBytes, { maxRows, maxBytes }, source) {
+function assertWithinModelRowLimit(rowCount, maxRows, source) {
   if (maxRows > 0 && rowCount > maxRows) {
     recordQueryCapExceeded("rows", source);
     throw new PayloadTooLargeError(`Query returned more than ${maxRows} rows. Refine the query (add a LIMIT or more selective WHERE) or raise PUBLISHER_MAX_QUERY_ROWS.`);
   }
-  if (maxBytes > 0 && serializedBytes > maxBytes) {
+}
+function assertWithinModelByteLimit(serialized, maxBytes, source) {
+  if (maxBytes <= 0)
+    return;
+  const serializedBytes = Buffer.byteLength(serialized, "utf8");
+  if (serializedBytes > maxBytes) {
     recordQueryCapExceeded("bytes", source);
     throw new PayloadTooLargeError(`Query response exceeded ${maxBytes} bytes (was ${serializedBytes}). Project fewer columns, add a LIMIT, or raise PUBLISHER_MAX_RESPONSE_BYTES.`);
   }
 }
-function stringifyQueryResponse(response, rowCount, maxBytes, source) {
+function stringifyQueryResponse(response, rowCount, maxBytes, source, replacer) {
   try {
-    return JSON.stringify(response);
+    return JSON.stringify(response, replacer);
   } catch (error) {
     if (!(error instanceof RangeError) || /call stack/i.test(error.message)) {
       throw error;
@@ -278087,6 +278170,15 @@ function stringifyQueryResponse(response, rowCount, maxBytes, source) {
   }
 }
 
+// src/json_utils.ts
+var MAX_SAFE_BIGINT = BigInt(Number.MAX_SAFE_INTEGER);
+function bigIntReplacer(_key, value) {
+  if (typeof value === "bigint") {
+    return value > MAX_SAFE_BIGINT || value < -MAX_SAFE_BIGINT ? value.toString() : Number(value);
+  }
+  return value;
+}
+
 // src/service/query_text.ts
 function extractRunTargetSourceName(query) {
   if (!query)
@@ -278150,12 +278242,12 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
         onParseError?.(sourceName, err);
       }
     }
-    const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
+    const ownNotes = ownLevelNoteTexts(struct.annotations);
     const ownGates = collectAuthorizeExprs(ownNotes);
     let inheritedGates = [];
     if (ownGates.length === 0) {
       for (let cur2 = struct.annotations?.inherits;cur2; cur2 = cur2.inherits) {
-        const exprs = collectAuthorizeExprs((cur2.blockNotes ?? []).map((note) => note.text));
+        const exprs = collectAuthorizeExprs(ownLevelNoteTexts(cur2));
         if (exprs.length > 0) {
           inheritedGates = exprs;
           break;
@@ -278417,7 +278509,7 @@ class Model {
     }
   }
   gateExprsForOwnAnnotations(struct, modelDef) {
-    const ownNotes = (struct.annotations?.blockNotes ?? []).map((note) => note.text);
+    const ownNotes = ownLevelNoteTexts(struct.annotations);
     try {
       const own = collectAuthorizeExprs(ownNotes);
       if (own.length > 0) {
@@ -278440,7 +278532,7 @@ class Model {
   ancestorGateExprs(struct, modelDef, seen = new Set) {
     let inherited = struct.annotations?.inherits;
     for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
-      const exprs2 = collectAuthorizeExprs((inherited.blockNotes ?? []).map((note) => note.text));
+      const exprs2 = collectAuthorizeExprs(ownLevelNoteTexts(inherited));
       if (exprs2.length > 0)
         return exprs2;
       inherited = inherited.inherits;
@@ -278455,7 +278547,7 @@ class Model {
       return ["false"];
     if (declared.kind === "none" || seen.has(declared.source))
       return [];
-    const exprs = collectAuthorizeExprs((declared.source.annotations?.blockNotes ?? []).map((note) => note.text));
+    const exprs = collectAuthorizeExprs(ownLevelNoteTexts(declared.source.annotations));
     return exprs.length > 0 ? exprs : this.ancestorGateExprs(declared.source, modelDef, seen);
   }
   resolveDeclaredSource(struct, modelDef) {
@@ -278798,7 +278890,7 @@ class Model {
         logger.warn(`Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`);
         for (const e of errors2) {
           findings.push({
-            target: target.label,
+            subject: target.label,
             message: e.message,
             severity: "error"
           });
@@ -278950,7 +279042,7 @@ class Model {
       return { ...b, schema, refinements };
     }).filter((b) => b.schema.length > 0);
   }
-  async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal, queryMetadataInput) {
+  async getQueryResults(sourceName, queryName, query, filterParams, bypassFilters, givens, abortSignal, queryMetadataInput, responseShape = "full") {
     const startTime = performance.now();
     if (this.compilationError) {
       if (this.compilationError instanceof MalloyError2 || this.compilationError instanceof ModelCompilationError) {
@@ -279167,10 +279259,10 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
       servedFrom = "live_fallback";
       executionTime = performance.now() - startTime;
     }
+    assertWithinModelRowLimit(queryResults.totalRows, maxRows, "model_query");
     const wrappedResult = API.util.wrapResult(queryResults);
-    const serializedResult = stringifyQueryResponse(wrappedResult, queryResults.totalRows, maxBytes, "model_query");
-    const serializedBytes = maxBytes > 0 ? Buffer.byteLength(serializedResult, "utf8") : 0;
-    assertWithinModelResponseLimits(queryResults.totalRows, serializedBytes, { maxRows, maxBytes }, "model_query");
+    const serializedResult = stringifyQueryResponse(responseShape === "compact" ? queryResults.data.value : wrappedResult, queryResults.totalRows, maxBytes, "model_query", responseShape === "compact" ? bigIntReplacer : undefined);
+    assertWithinModelByteLimit(serializedResult, maxBytes, "model_query");
     this.queryExecutionHistogram.record(executionTime, {
       "malloy.model.path": this.modelPath,
       "malloy.model.query.name": queryName,
@@ -279250,8 +279342,8 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
         queryInfo: cell.queryInfo ? JSON.stringify(cell.queryInfo) : undefined
       };
     });
-    const allAnnotations = this.modelDef ? new Annotations2(modelAnnotations(this.modelDef)).texts() : [];
-    return {
+    const allAnnotations = this.modelDef ? ownModelNotes(this.modelDef) : [];
+    const notebook = {
       type: "notebook",
       packageName: this.packageName,
       modelPath: this.modelPath,
@@ -279262,6 +279354,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
       annotations: allAnnotations,
       notebookCells
     };
+    return notebook;
   }
   async executeNotebookCell(cellIndex, filterParams, bypassFilters, givens, abortSignal, queryMetadataInput) {
     if (this.compilationError) {
@@ -279322,9 +279415,12 @@ run: ${sourceName ? `${quoteMalloyIdentifier(sourceName)} -> ` : ""}${quoteMallo
         });
         const query = (await runnableToExecute.getPreparedQuery())._query;
         queryName = query.as || query.name;
+        if (result?._queryResult) {
+          assertWithinModelRowLimit(result.totalRows, cellMaxRows, "notebook_cell");
+        }
         queryResult = result?._queryResult && this.modelInfo && stringifyQueryResponse(API.util.wrapResult(result), result.totalRows, cellMaxBytes, "notebook_cell");
         if (result?._queryResult && queryResult) {
-          assertWithinModelResponseLimits(result.totalRows, Buffer.byteLength(queryResult, "utf8"), { maxRows: cellMaxRows, maxBytes: cellMaxBytes }, "notebook_cell");
+          assertWithinModelByteLimit(queryResult, cellMaxBytes, "notebook_cell");
         }
       } catch (error) {
         if (error instanceof FilterValidationError) {
@@ -279788,7 +279884,7 @@ function detectDroppedPersistSources(malloyModel, recognizedNames) {
         continue;
       let isPersist = false;
       try {
-        isPersist = new Annotations3(def.annotations).parseAsTag("@").tag.has("persist");
+        isPersist = new Annotations2(def.annotations).parseAsTag("@").tag.has("persist");
       } catch {
         continue;
       }
@@ -279915,7 +280011,7 @@ function collectSourceEligibility(sources) {
 }
 
 // src/service/materialization_config_validation.ts
-function metadataWarnings(level, metadata, target) {
+function metadataWarnings(level, metadata, subject) {
   if (!metadata)
     return [];
   const budget = queryMetadataBudgetWarning(Object.keys(metadata).length);
@@ -279925,7 +280021,7 @@ function metadataWarnings(level, metadata, target) {
     ...budget ? [budget] : []
   ].map((message) => ({
     message: `${level}: ${message}`,
-    ...target ? { target } : {}
+    ...subject ? { subject } : {}
   }));
 }
 function materializationConfigWarnings(input) {
@@ -279936,7 +280032,7 @@ function materializationConfigWarnings(input) {
   }
   const seen = new Set;
   return warnings.filter((warning) => {
-    const key = `${warning.target ?? ""}\x00${warning.message}`;
+    const key = `${warning.subject ?? ""}\x00${warning.message}`;
     if (seen.has(key))
       return false;
     seen.add(key);
@@ -280211,7 +280307,12 @@ class Package {
         }
         const model = Model.fromSerialized(packageName, packagePath, malloyConfig, sm);
         for (const w of await model.validateRenderTags()) {
-          renderTagWarnings.push({ model: sm.modelPath, ...w });
+          renderTagWarnings.push({
+            model: sm.modelPath,
+            subject: w.subject,
+            message: w.message,
+            severity: w.severity
+          });
         }
         if (sm.modelPath.endsWith(MODEL_FILE_SUFFIX)) {
           const modelSource = await fs7.readFile(path7.join(packagePath, sm.modelPath), "utf-8");
@@ -280438,7 +280539,7 @@ class Package {
   droppedPersistWarnings() {
     return this.droppedPersistSources.map((d) => ({
       model: d.modelPath,
-      target: d.name,
+      subject: d.name,
       message: `is annotated '#@ persist' but was not recognized as a ` + `materializable source, so nothing is materialized and it is served ` + `live. Only query/aggregate sources build; a filtered pass-through ` + `does not. Persist a query source, or invoke a parameterized source ` + `with a bound argument.`
     }));
   }
@@ -280454,7 +280555,7 @@ class Package {
       const message = mode === "off" ? `declares storage="${storage}" but PERSIST_STORAGE_MODE is off; ` + `the annotation is ignored and the source is served live from ` + `its own warehouse.` : `is materialized into storage "${storage}" but ` + `PERSIST_STORAGE_MODE is write-only; the serve path is not ` + `routed to the materialized table (served live).`;
       warnings.push({
         model: source.modelPath ?? "",
-        target: source.name,
+        subject: source.name,
         message
       });
     }
@@ -280633,7 +280734,12 @@ class Package {
         const model = Model.fromSerialized(this.packageName, this.packagePath, this.malloyConfig, sm, { buildManifest });
         try {
           for (const w of await model.validateRenderTags()) {
-            renderTagWarnings.push({ model: sm.modelPath, ...w });
+            renderTagWarnings.push({
+              model: sm.modelPath,
+              subject: w.subject,
+              message: w.message,
+              severity: w.severity
+            });
           }
           nextModels.set(sm.modelPath, model);
         } catch (renderErr) {
@@ -286167,7 +286273,7 @@ var BACK_PRESSURE_SUGGESTIONS = [
   "If this persists, raise the limit or scale up the pod."
 ];
 function classifyToolError(operation, identifier, error) {
-  if (error instanceof EnvironmentNotFoundError || error instanceof PackageNotFoundError || error instanceof ModelNotFoundError || error instanceof NotQueryableError) {
+  if (error instanceof EnvironmentNotFoundError || error instanceof PackageNotFoundError || error instanceof ModelNotFoundError || error instanceof NotQueryableError || error instanceof ConnectionNotFoundError) {
     return getNotFoundError(identifier);
   }
   if (error instanceof ServiceUnavailableError) {
@@ -286194,6 +286300,15 @@ function classifyToolError(operation, identifier, error) {
       ]
     };
   }
+  if (error instanceof InvalidArgumentError) {
+    return {
+      message: error.message,
+      suggestions: [
+        "This is not transient. The same arguments will fail the same way, so change them rather than retrying.",
+        "The message above names what was wrong. If it names an expected format, use that format exactly."
+      ]
+    };
+  }
   if (error instanceof MalloyError5 || error instanceof ModelCompilationError || error instanceof AccessDeniedError || error instanceof BadRequestError) {
     return getMalloyErrorDetails(operation, identifier, error);
   }
@@ -286632,7 +286747,7 @@ function registerExecuteQueryTool(mcpServer, environmentStore) {
         rowLimit,
         rowLimitSource,
         queryCorrelationId
-      } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal, queryMetadataInput) : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal, queryMetadataInput), getQueryTimeoutMs());
+      } = await runWithQueryTimeout((abortSignal) => query ? model.getQueryResults(undefined, undefined, query, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact") : model.getQueryResults(sourceName, queryName, undefined, filterParams, undefined, givens, abortSignal, queryMetadataInput, "compact"), getQueryTimeoutMs());
       const { validateRenderTags: validateRenderTags2 } = await Promise.resolve().then(() => __toESM(require_dist11(), 1));
       const renderLogs = validateRenderTags2(result);
       const resultUri = buildMalloyUri({
@@ -287033,6 +287148,579 @@ function registerReloadPackageTool(mcpServer, environmentStore) {
     }
   });
 }
+
+// src/mcp/tools/search_database_schema_tool.ts
+init_logger();
+
+// src/mcp/tools/schema_index.ts
+init_logger();
+var import_lunr3 = __toESM(require_lunr(), 1);
+import { createHash as createHash2 } from "crypto";
+var MAX_INDEXED_TABLES = 5000;
+var MAX_COLUMNS_IN_INDEX_TEXT = 60;
+var MAX_CACHED_SCHEMAS = 8;
+var PROVIDER_FAILURE_COOLDOWN_MS2 = 60000;
+function tableIndexText(entity) {
+  const name = humanizeName(entity.tableName) || entity.tableName;
+  const schema = humanizeName(entity.schemaName) || entity.schemaName;
+  const columns = entity.columns.slice(0, MAX_COLUMNS_IN_INDEX_TEXT);
+  const columnNames = columns.map((c) => humanizeName(c.name) || c.name).filter(Boolean).join(", ");
+  const types2 = Array.from(new Set(columns.map((c) => c.type).filter((t) => Boolean(t)))).join(" ");
+  return [schema, name, columnNames, types2].filter(Boolean).join(" ");
+}
+function schemaFingerprint(tables) {
+  const canonical = tables.map((t) => `${t.resource}\x00${t.columns.map((c) => `${c.name}:${c.type ?? ""}`).join(",")}`).sort().join("\x01");
+  return createHash2("sha256").update(canonical).digest("hex");
+}
+function sanitizeQuery(query) {
+  return query.replace(/[~^:*+\-"]/g, " ").trim();
+}
+function rankLexically(tables, query, limit, cacheKey) {
+  const sanitized = sanitizeQuery(query);
+  if (!sanitized || tables.length === 0)
+    return { hits: [], matched: 0 };
+  const fingerprint = cacheKey ? schemaFingerprint(tables) : "";
+  let cached2 = cacheKey ? lexicalCache.get(cacheKey) : undefined;
+  if (cached2 && cached2.fingerprint !== fingerprint)
+    cached2 = undefined;
+  const byResource = cached2?.byResource ?? new Map(tables.map((t) => [t.resource, t]));
+  const index = cached2?.index ?? import_lunr3.default(function() {
+    this.ref("resource");
+    this.field("text");
+    this.metadataWhitelist = [];
+    for (const table of tables) {
+      this.add({ resource: table.resource, text: tableIndexText(table) });
+    }
+  });
+  if (cacheKey && !cached2) {
+    lexicalCache.delete(cacheKey);
+    lexicalCache.set(cacheKey, { index, byResource, fingerprint });
+    while (lexicalCache.size > MAX_CACHED_SCHEMAS) {
+      const oldest = lexicalCache.keys().next();
+      if (oldest.done)
+        break;
+      lexicalCache.delete(oldest.value);
+    }
+  } else if (cacheKey && cached2) {
+    lexicalCache.delete(cacheKey);
+    lexicalCache.set(cacheKey, cached2);
+  }
+  let hits;
+  try {
+    hits = index.search(sanitized);
+  } catch (error) {
+    logger.warn("[MCP Tool searchDatabaseSchema] lunr search failed", {
+      error: error instanceof Error ? error.message : String(error)
+    });
+    return { hits: [], matched: 0 };
+  }
+  return {
+    hits: hits.slice(0, limit).map((hit) => {
+      const table = byResource.get(hit.ref);
+      return table ? { ...table, score: hit.score } : undefined;
+    }).filter((t) => t !== undefined),
+    matched: hits.length
+  };
+}
+function cosineSimilarity(a, b) {
+  if (a.length !== b.length || a.length === 0)
+    return 0;
+  let dot = 0;
+  let magA = 0;
+  let magB = 0;
+  for (let i = 0;i < a.length; i++) {
+    dot += a[i] * b[i];
+    magA += a[i] * a[i];
+    magB += b[i] * b[i];
+  }
+  if (magA === 0 || magB === 0)
+    return 0;
+  return dot / (Math.sqrt(magA) * Math.sqrt(magB));
+}
+var vectorCache = new Map;
+var lexicalCache = new Map;
+var cooldownUntilMs = new Map;
+function touchCacheEntry(key) {
+  const entry = vectorCache.get(key);
+  if (entry) {
+    vectorCache.delete(key);
+    vectorCache.set(key, entry);
+  }
+  return entry;
+}
+function storeCacheEntry(key, entry) {
+  vectorCache.delete(key);
+  vectorCache.set(key, entry);
+  while (vectorCache.size > MAX_CACHED_SCHEMAS) {
+    const oldest = vectorCache.keys().next();
+    if (oldest.done)
+      break;
+    vectorCache.delete(oldest.value);
+  }
+}
+function pruneCooldowns() {
+  if (cooldownUntilMs.size <= MAX_CACHED_SCHEMAS * 4)
+    return;
+  const now = Date.now();
+  for (const [k, until] of cooldownUntilMs) {
+    if (until <= now)
+      cooldownUntilMs.delete(k);
+  }
+}
+function providerKeyFor(provider) {
+  return `${provider.model}\x00${provider.dimensions ?? ""}`;
+}
+async function tryRankSemantically(args) {
+  const { tables, query, limit, provider, cacheKey } = args;
+  if (tables.length === 0) {
+    return null;
+  }
+  if (tables.length > MAX_INDEXED_TABLES) {
+    logger.warn("[MCP Tool searchDatabaseSchema] Schema exceeds the semantic index cap; ranking lexically", { tableCount: tables.length, cap: MAX_INDEXED_TABLES });
+    return null;
+  }
+  const until = cooldownUntilMs.get(cacheKey);
+  if (until !== undefined && Date.now() < until)
+    return null;
+  const fingerprint = schemaFingerprint(tables);
+  const providerKey = providerKeyFor(provider);
+  let entry = touchCacheEntry(cacheKey);
+  try {
+    if (!entry || entry.fingerprint !== fingerprint || entry.providerKey !== providerKey) {
+      const texts = tables.map(tableIndexText);
+      const vectors = await provider.embedBatch(texts, EMBEDDING_BATCH_TIMEOUT_MS);
+      if (vectors.length !== tables.length) {
+        throw new Error(`Embedding provider returned ${vectors.length} vectors for ${tables.length} inputs`);
+      }
+      entry = {
+        vectors: new Map(tables.map((t, i) => [t.resource, vectors[i]])),
+        fingerprint,
+        providerKey
+      };
+      storeCacheEntry(cacheKey, entry);
+    }
+    const [queryVector] = await provider.embedBatch([query], EMBEDDING_QUERY_TIMEOUT_MS);
+    if (!queryVector)
+      throw new Error("Empty query embedding");
+    const scored = [];
+    for (const table of tables) {
+      const vector = entry.vectors.get(table.resource);
+      if (!vector)
+        continue;
+      const score = cosineSimilarity(queryVector, vector);
+      if (score >= MIN_SIMILARITY)
+        scored.push({ ...table, score });
+    }
+    scored.sort((a, b) => b.score - a.score);
+    return { hits: scored.slice(0, limit), matched: scored.length };
+  } catch (error) {
+    cooldownUntilMs.set(cacheKey, Date.now() + PROVIDER_FAILURE_COOLDOWN_MS2);
+    pruneCooldowns();
+    logger.warn("[MCP Tool searchDatabaseSchema] Embedding failed; ranking lexically", {
+      cacheKey,
+      error: error instanceof Error ? error.message : String(error)
+    });
+    return null;
+  }
+}
+async function rankTables(args) {
+  const { tables, query, limit, provider, cacheKey } = args;
+  if (!sanitizeQuery(query)) {
+    return { hits: [], matched: 0, ranking: "lexical", emptyQuery: true };
+  }
+  if (provider) {
+    const semantic = await tryRankSemantically({
+      tables,
+      query,
+      limit,
+      provider,
+      cacheKey
+    });
+    if (semantic !== null)
+      return { ...semantic, ranking: "semantic" };
+  }
+  return {
+    ...rankLexically(tables, query, limit, cacheKey),
+    ranking: "lexical"
+  };
+}
+
+// src/mcp/tools/search_database_schema_tool.ts
+var DEFAULT_LIMIT = 20;
+var MAX_LIMIT = 100;
+var MAX_COLUMNS_PER_TABLE = 80;
+var MAX_ARG_CHARS = 256;
+var searchDatabaseSchemaShape = {
+  environmentName: exports_external.string().max(MAX_ARG_CHARS).optional().describe("Environment to look in. Omit to list the environments and their connections."),
+  connectionName: exports_external.string().max(MAX_ARG_CHARS).optional().describe("Connection to introspect. Omit to list the connections in the environment."),
+  packageName: exports_external.string().max(MAX_ARG_CHARS).optional().describe('Required only for the per-package "duckdb" sandbox connection, which exists once per package.'),
+  schemaName: exports_external.string().max(MAX_ARG_CHARS).optional().describe(`Schema (or dataset/database) to list tables from. Omit to list the connection's schemas and use one of those names verbatim: DuckDB qualifies them as "catalog.schema" (for example "memory.main"), so a bare "main" is rejected.`),
+  tableName: exports_external.string().max(MAX_ARG_CHARS).optional().describe("A single table to return in full, with every column. Requires schemaName."),
+  searchQuery: exports_external.string().max(500).optional().describe(`Plain-English description of the data you are looking for, e.g. "customer orders and shipping addresses". Requires schemaName. Omit to list the schema's tables in order.`),
+  limit: exports_external.number().int().positive().max(MAX_LIMIT).optional().describe(`Maximum tables to return. Default ${DEFAULT_LIMIT}.`),
+  offset: exports_external.number().int().min(0).optional().describe("Tables to skip, for paging a long table listing. Pass back the nextOffset from a previous response. Ignored when searchQuery is set, because ranked results cannot be paged.")
+};
+function escapeMalloyString2(value) {
+  return value.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
+}
+function malloyIdentifier(name) {
+  return `\`${name.replace(/\\/g, "\\\\").replace(/`/g, "\\`")}\``;
+}
+function malloySourceSnippet(connectionName, resource, tableName) {
+  const alias = malloyIdentifier(tableName);
+  const connection = malloyIdentifier(connectionName);
+  return `source: ${alias} is ${connection}.table('${escapeMalloyString2(resource)}') extend { }`;
+}
+var DUCKDB_FILE_PATH = /^[A-Za-z0-9._~:/?#@!$&*+,=%-]+$/;
+var STRICT_BARE_IDENT = /^[A-Za-z_][A-Za-z0-9_]*$/;
+function isPastableTablePath(resource) {
+  if (!resource)
+    return false;
+  if (resource.includes("/"))
+    return DUCKDB_FILE_PATH.test(resource);
+  if (DATA_FILE_EXTENSION.test(resource))
+    return false;
+  return resource.split(".").every((seg) => STRICT_BARE_IDENT.test(seg));
+}
+function canPasteSource(entity) {
+  return Boolean(entity.tableName) && isPastableTablePath(entity.resource);
+}
+var DATA_FILE_EXTENSION = /\.(parquet|csv|tsv|json|jsonl|ndjson|xlsx)$/i;
+function bareTableName2(resource) {
+  const lastSlash = resource.lastIndexOf("/");
+  if (lastSlash >= 0) {
+    return resource.slice(lastSlash + 1).replace(DATA_FILE_EXTENSION, "");
+  }
+  if (DATA_FILE_EXTENSION.test(resource)) {
+    return resource.replace(DATA_FILE_EXTENSION, "");
+  }
+  return resource.slice(resource.lastIndexOf(".") + 1);
+}
+var SANDBOX_CONNECTION = "duckdb";
+async function listConnectionsFor(environment) {
+  const connections = environment.listApiConnections().map((c) => ({ name: c.name ?? "", type: c.type ?? "" }));
+  let packages = [];
+  try {
+    packages = (await environment.listPackages()).map((p) => p.name).filter((n) => Boolean(n));
+  } catch {
+    packages = [];
+  }
+  if (packages.length > 0) {
+    connections.push({
+      name: SANDBOX_CONNECTION,
+      type: "duckdb",
+      scope: "package",
+      packages
+    });
+  }
+  return connections;
+}
+function toResponseTable(entity, options) {
+  return {
+    connectionName: entity.connectionName,
+    schemaName: entity.schemaName,
+    tableName: entity.tableName,
+    tablePath: entity.resource,
+    ...canPasteSource(entity) ? {
+      malloySource: malloySourceSnippet(entity.connectionName, entity.resource, entity.tableName)
+    } : {},
+    columns: entity.columns.slice(0, options.maxColumns),
+    columnCount: entity.columns.length,
+    ...options.score !== undefined ? { score: options.score } : {}
+  };
+}
+var SEARCH_DATABASE_SCHEMA_DESCRIPTION = `Find the tables in a database connection, by plain-English description. Use it to model a database you have not modelled yet, or to check schema, table and column names. To search an existing model, use malloy_getContext.
+
+## Drill down, one level at a time
+Supply what you know, omit the rest. No arguments lists the environments and their connections; + connectionName lists its schemas; + schemaName lists its tables (up to ${MAX_COLUMNS_PER_TABLE} columns each; add searchQuery to rank them); + tableName returns that one table with every column.
+
+## Contract rules
+- Use connectionName, tablePath and column names exactly as returned.
+- A connection with scope "package" (the "duckdb" sandbox) is per package: pass packageName too, from those it lists.
+- Schemas marked isHidden are system schemas; your tables are in the others.
+- malloySource is the ready-to-use \`source:\` line; its identifiers are already quoted, so paste it as-is.
+- Names and types only: no row value is returned. For a column's values, run malloy_executeQuery against a model using this connection: \`run: c.table('s.t') -> { group_by: col }\`.
+- No tables for a searchQuery means nothing matched, not an empty schema. Broaden it, or list without one.
+- An empty schema may still hold data: DuckDB over CSV or Parquet addresses files by path, registering none.
+- Read warnings: they name anything omitted or ignored.
+
+## Response
+JSON: tables (connectionName, schemaName, tableName, tablePath, malloySource, columns, columnCount, and score on a search), plus totalAvailable and returned. A search adds matched and ranking; a listing adds nextOffset when more remain, to pass back as offset.
+
+## Worked example
+Start with no arguments and follow what it names. For connection "warehouse", schema "sales":
+{ "environmentName": "examples", "connectionName": "warehouse", "schemaName": "sales", "searchQuery": "customer orders" }
+Then paste that table's malloySource verbatim, e.g. source: \`orders\` is \`warehouse\`.table('sales.orders') extend { }`;
+function registerSearchDatabaseSchemaTool(mcpServer, environmentStore) {
+  const connectionController = new ConnectionController(environmentStore);
+  const unpastableWarning = (entities) => {
+    const n = entities.filter((e) => !canPasteSource(e)).length;
+    if (n === 0)
+      return [];
+    return [
+      `${n} table(s) have a path this server will not vouch for across every dialect it serves, so malloySource is omitted for them rather than risk a line that does not compile. Build it from tablePath: on your own dialect the path may work as-is, or may need the offending segment quoted. Double quotes on DuckDB, Postgres, Snowflake and Trino; backticks on MySQL, BigQuery and Databricks.`
+    ];
+  };
+  const assertCanAdmit = async (environmentName) => {
+    (await environmentStore.getEnvironment(environmentName, false)).assertCanAdmitQuery();
+  };
+  mcpServer.tool("malloy_searchDatabaseSchema", SEARCH_DATABASE_SCHEMA_DESCRIPTION, searchDatabaseSchemaShape, async (params) => {
+    const {
+      environmentName,
+      connectionName,
+      packageName,
+      schemaName,
+      tableName,
+      searchQuery,
+      limit,
+      offset
+    } = params;
+    const max = limit ?? DEFAULT_LIMIT;
+    const skip = offset ?? 0;
+    const uri = buildMalloyUri({
+      environment: environmentName,
+      package: packageName
+    }, "search-database-schema");
+    logger.info("[MCP Tool searchDatabaseSchema] Searching schema", {
+      environmentName,
+      connectionName,
+      packageName,
+      schemaName,
+      tableName,
+      searchQuery,
+      limit,
+      offset
+    });
+    try {
+      const ignored = [];
+      const noteIgnored = (entries) => {
+        for (const [name, value] of entries) {
+          if (value !== undefined && value !== "" && !ignored.includes(name)) {
+            ignored.push(name);
+          }
+        }
+      };
+      const pagingArgs = [
+        ["limit", limit],
+        ["offset", offset]
+      ];
+      if (!environmentName) {
+        noteIgnored([
+          ["connectionName", connectionName],
+          ["packageName", packageName],
+          ["schemaName", schemaName],
+          ["tableName", tableName],
+          ["searchQuery", searchQuery],
+          ...pagingArgs
+        ]);
+      } else if (!connectionName) {
+        noteIgnored([
+          ["packageName", packageName],
+          ["schemaName", schemaName],
+          ["tableName", tableName],
+          ["searchQuery", searchQuery],
+          ...pagingArgs
+        ]);
+      } else if (!schemaName) {
+        noteIgnored([
+          ["tableName", tableName],
+          ["searchQuery", searchQuery],
+          ...pagingArgs
+        ]);
+      } else if (tableName) {
+        noteIgnored([["searchQuery", searchQuery], ...pagingArgs]);
+      } else if (searchQuery !== undefined) {
+        noteIgnored([["offset", offset]]);
+      }
+      if (connectionName && connectionName !== SANDBOX_CONNECTION) {
+        noteIgnored([["packageName", packageName]]);
+      }
+      const ignoredWarning = ignored.length > 0 ? [
+        `Ignored ${ignored.join(", ")}. Each level needs the one above it: environmentName, then connectionName (with packageName for the per-package "duckdb" sandbox), then schemaName, then either tableName or searchQuery. limit and offset apply only to a plain table listing, and offset does nothing on a ranked search because ranked results cannot be paged.`
+      ] : [];
+      if (!environmentName) {
+        const environments = await environmentStore.listEnvironments();
+        const results = await Promise.all(environments.map(async (env) => {
+          const name = env.name ?? "";
+          let connections = [];
+          try {
+            const environment = await environmentStore.getEnvironment(name, false);
+            connections = await listConnectionsFor(environment);
+          } catch (error) {
+            logger.debug("[MCP Tool searchDatabaseSchema] Could not list connections", {
+              environmentName: name,
+              error: error instanceof Error ? error.message : String(error)
+            });
+          }
+          return { name, connections };
+        }));
+        return jsonResource(uri, {
+          environments: results,
+          ...ignoredWarning.length > 0 ? { warnings: ignoredWarning } : {}
+        });
+      }
+      if (!connectionName) {
+        const environment = await environmentStore.getEnvironment(environmentName, false);
+        const connections = await listConnectionsFor(environment);
+        return jsonResource(uri, {
+          environmentName,
+          connections,
+          ...ignoredWarning.length > 0 ? { warnings: ignoredWarning } : {}
+        });
+      }
+      if (!schemaName) {
+        await assertCanAdmit(environmentName);
+        const schemas = await connectionController.listSchemas(environmentName, connectionName, packageName);
+        return jsonResource(uri, {
+          environmentName,
+          connectionName,
+          schemas: schemas.map((s) => ({
+            name: s.name ?? "",
+            isDefault: s.isDefault ?? false,
+            isHidden: s.isHidden ?? false
+          })),
+          ...ignoredWarning.length > 0 ? { warnings: ignoredWarning } : {}
+        });
+      }
+      if (tableName) {
+        await assertCanAdmit(environmentName);
+        const tables = await connectionController.listTables(environmentName, connectionName, schemaName, [tableName], packageName);
+        if (tables.length === 0) {
+          return jsonToolError(uri, {
+            message: `Table "${tableName}" not found in schema "${schemaName}" of connection "${connectionName}".`,
+            suggestions: [
+              `List the schema's tables by calling this tool with schemaName "${schemaName}" and no tableName.`,
+              "Check the table name's spelling and case; some warehouses are case-sensitive."
+            ]
+          }, { tables: [] });
+        }
+        const exact = tables.filter((t) => bareTableName2(t.resource ?? "") === tableName);
+        const matching = exact.length > 0 ? exact : tables.filter((t) => bareTableName2(t.resource ?? "").toLowerCase() === tableName.toLowerCase());
+        const entities2 = (matching.length > 0 ? matching : tables).map((t) => toEntity(t, connectionName, schemaName));
+        if (matching.length === 0) {
+          return jsonToolError(uri, {
+            message: `Table "${tableName}" not found in schema "${schemaName}" of connection "${connectionName}".`,
+            suggestions: [
+              `This schema lists ${tables.length} table(s). Call this tool with schemaName "${schemaName}" and no tableName to see them.`,
+              "Check the table name's spelling and case; some warehouses are case-sensitive."
+            ]
+          }, { tables: [] });
+        }
+        return jsonResource(uri, {
+          environmentName,
+          connectionName,
+          schemaName,
+          tables: entities2.map((e) => toResponseTable(e, {
+            maxColumns: Number.MAX_SAFE_INTEGER
+          })),
+          totalAvailable: entities2.length,
+          returned: entities2.length,
+          ...(() => {
+            const w = [
+              ...ignoredWarning,
+              ...unpastableWarning(entities2)
+            ];
+            return w.length > 0 ? { warnings: w } : {};
+          })()
+        });
+      }
+      await assertCanAdmit(environmentName);
+      const allTables = await connectionController.listTables(environmentName, connectionName, schemaName, undefined, packageName);
+      const entities = allTables.map((t) => toEntity(t, connectionName, schemaName));
+      const warnings = [...ignoredWarning];
+      if (entities.length === 0) {
+        warnings.push(`No tables are registered in schema "${schemaName}". If this connection reads data files directly (DuckDB over CSV or Parquet), those files are not listed in a schema; reference them by path instead, for example ${connectionName}.table('data/orders.parquet'). Otherwise check the schema name against the schema list for this connection.`);
+      }
+      let page;
+      let ranking;
+      let nextOffset;
+      let matched;
+      if (searchQuery !== undefined) {
+        const provider = resolveProvider();
+        const ranked = await rankTables({
+          tables: entities,
+          query: searchQuery,
+          limit: max,
+          provider,
+          cacheKey: [
+            environmentName,
+            connectionName === SANDBOX_CONNECTION ? packageName ?? "" : "",
+            connectionName,
+            schemaName
+          ].join("\x00")
+        });
+        ranking = ranked.emptyQuery ? undefined : ranked.ranking;
+        matched = ranked.emptyQuery ? undefined : ranked.matched;
+        page = ranked.hits.map((hit) => ({
+          entity: hit,
+          score: hit.score
+        }));
+        if (ranked.emptyQuery) {
+          warnings.push(`searchQuery carried no searchable content, so no ranking was run. Provide words to search for, or omit searchQuery to list the schema's tables.`);
+        } else if (ranked.hits.length === 0 && entities.length > 0) {
+          warnings.push(`No table in "${schemaName}" matched "${searchQuery}". List the schema without a searchQuery to see everything in it.`);
+        }
+        if (ranked.matched > page.length) {
+          warnings.push(ranked.ranking === "lexical" ? `${ranked.matched} tables share at least one term with "${searchQuery}"; the top ${page.length} by relevance are shown. Term-matching counts loosely, so a large number here is normal and does not mean that many tables are relevant. Ranked results cannot be paged: make searchQuery more specific, or raise limit (max ${MAX_LIMIT}) if you want more of this ranking.` : `${ranked.matched} tables scored above the relevance floor for "${searchQuery}"; the top ${page.length} are shown. Ranked results cannot be paged: make searchQuery more specific, or raise limit (max ${MAX_LIMIT}).`);
+        }
+      } else {
+        page = entities.slice(skip, skip + max).map((entity) => ({ entity }));
+        if (skip + max < entities.length) {
+          nextOffset = skip + max;
+          warnings.push(`Showing ${page.length} of ${entities.length} tables. Pass offset ${nextOffset} for the next page, or add a searchQuery to narrow.`);
+        } else if (page.length === 0 && entities.length > 0) {
+          warnings.push(`offset ${skip} is past the end of this schema, which has ${entities.length} tables. Use an offset below ${entities.length}, or omit it to start from the beginning.`);
+        }
+      }
+      warnings.push(...unpastableWarning(page.map(({ entity }) => entity)));
+      const capped = page.filter(({ entity }) => entity.columns.length > MAX_COLUMNS_PER_TABLE).length;
+      if (capped > 0) {
+        warnings.push(`${capped} table(s) have more than ${MAX_COLUMNS_PER_TABLE} columns; only the first ${MAX_COLUMNS_PER_TABLE} are shown. Pass tableName to see all columns of one table.`);
+      }
+      return jsonResource(uri, {
+        environmentName,
+        connectionName,
+        schemaName,
+        tables: page.map(({ entity, score }) => toResponseTable(entity, {
+          maxColumns: MAX_COLUMNS_PER_TABLE,
+          score
+        })),
+        totalAvailable: entities.length,
+        returned: page.length,
+        ...matched !== undefined ? { matched } : {},
+        ...nextOffset !== undefined ? { nextOffset } : {},
+        ...ranking ? { ranking } : {},
+        ...warnings.length > 0 ? { warnings } : {}
+      });
+    } catch (error) {
+      const identifier = [environmentName, connectionName, schemaName].filter(Boolean).join("/");
+      return jsonToolError(uri, classifyToolError("searchDatabaseSchema", identifier, error), { tables: [] });
+    }
+  });
+}
+function resolveProvider() {
+  try {
+    if (!schemaEmbeddingEnabled())
+      return null;
+    return getEmbeddingProvider();
+  } catch (error) {
+    logger.warn("[MCP Tool searchDatabaseSchema] Embedding config invalid; ranking lexically", { error: error instanceof Error ? error.message : String(error) });
+    return null;
+  }
+}
+function toEntity(table, connectionName, schemaName) {
+  const resource = table.resource ?? "";
+  return {
+    connectionName,
+    schemaName,
+    tableName: bareTableName2(resource),
+    resource,
+    columns: (table.columns ?? []).map((c) => ({
+      name: c.name ?? "",
+      ...c.type ? { type: c.type } : {}
+    }))
+  };
+}
 // src/mcp/skills/skills_bundle.json
 var skills_bundle_default = {
   skills: [{ name: "malloy", description: 'Index of all Malloy skills. Use when user asks "malloy help", "what malloy skills are available", "how do I use malloy", or needs guidance on which Malloy skill to use.', body: `# Malloy Skills Index
@@ -287709,7 +288397,7 @@ Once the definitions are confirmed, write them into the \`.malloy\` model (see y
 
 ## Output
 
-A confirmed source architecture and a confirmed set of field definitions (renames, dimensions, measures, business decisions), held in the conversation and ready to write into the \`.malloy\` model via your modeling workflow.` }, { name: "malloy-discover", description: "Silent data discovery for Malloy modeling. Used at Step 1 of the modeling workflow. Scans tables, columns, distributions, and relationships without user interaction. The agent builds an internal picture before presenting anything.", body: "# Data Discovery (Step 1, Silent)\n\n> **CRITICAL**: Read the model before writing ANY Malloy code. The model defines the sources, connection names, and fields. Never guess connection names.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n> **PREREQUISITE:** Make sure the Malloy MCP tools (`get_context`, `execute_query`, `search_malloy_docs`) are configured and reachable. If they are not, stop and resolve the MCP connection before continuing.\n\n**This step is silent.** The agent does not present findings to the user yet. That happens in the next step (PROPOSE SCOPE).\n\n## Tools\n\n- **`get_context`**: Ground yourself in the package's sources, views, and fields (with their docs). Call FIRST. The sources and their join paths are the schema you build on.\n- **`execute_query`**: Run ad-hoc queries to preview data, verify values, check NULLs, validate assumptions.\n- **`search_malloy_docs`**: Get Malloy syntax help when needed.\n\n## Workflow\n\n```\n1. Check for prior art signals                 → If found, ask user: \"I found [LookML/dbt] files, use as prior art?\"\n2. If user confirms: read adapter reference        → Follow skill:malloy-lookml-review, keep prior-art notes in-conversation\n3. get_context                           → Ground yourself: sources, views, fields\n4. Inspect source definitions                  → See ALL fields and join paths for key sources\n5. Derive candidate joins/dimensions/measures  → Read them off the model and the data, not a suggestion tool\n6. Define a minimal source if one is missing   → Just enough to run execute_query for previews\n7. execute_query(query)                  → Preview data, verify values, check NULLs, check duplicates\n8. search_malloy_docs(query)                    → Get syntax help when needed\n9. Proceed to Step 2 (PROPOSE SCOPE)\n```\n\n**If the model has no sources defined** and no LookML files are present, do NOT silently retry or proceed without data. Tell the user: \"No model sources were found. Please check that the package points at a connected data source, then try again.\"\n\n**If the model has no sources defined** but LookML files ARE present (LookML-only mode), skip steps 3-7. Use connection name and table paths from the LookML review. Flag all proposals as unvalidated.\n\n**Key principle:** Query data to verify assumptions. Don't ask the user to confirm values you can check yourself.\n\n**Search docs proactively.** If you discover patterns that need derived/pre-aggregated sources, window functions, or unfamiliar features, call `search_malloy_docs` BEFORE writing code, not just when you hit errors.\n\n## Query File for Discovery\n\n**In the schema-first workflow:** Run ad-hoc queries with `execute_query`. If the source you want to preview is not yet defined in the model, define a minimal one against the connection and table so you can run previews. The real model fields are built in later steps.\n\n```malloy\n// minimal source for previewing data during discovery\nsource: explore is my_conn.table('schema.table') extend {}\n```\n\n**In analysis-first mode:** There is no temp file. The analysis `.malloy` file IS your working file. It grows throughout the session and becomes the input for formalizing into a model. See `skill:malloy-analyze` for that workflow.\n\n## What to Capture\n\nWhen reviewing tables and columns, capture:\n\n### Table-Level\n- All tables with row counts\n- Connection name and schema (CRITICAL, never guess)\n- Table roles: fact, dimension, bridge, lookup, staging, operational\n- Join relationships (FK → PK mappings)\n\n### Column-Level\n- Primary key and foreign key columns\n- Data types (watch for string dates, arrays, JSON)\n- Reserved word columns that need backticking (`Date`, `Type`, `number`, `source`, etc.)\n- Column cardinality and NULL rates (via `execute_query`)\n- Data distributions for key numeric and categorical columns\n\n### Data Quality\n- **Check for duplicate rows** on primary keys. Run `group_by: pk, aggregate: count(), having: count() > 1` on each key table. Duplicates cause `sum()` to return nonsensical values.\n- **Denormalized count columns**: beware pre-aggregated fields (e.g., `order_count` in a customer table) that may conflict with joined counts.\n- **Delimited list columns**: flag string columns containing comma-separated values.\n\n### Data-Driven Validation\n\n**Every recommendation must be grounded in queried data, not schema inference.** During discovery, run `execute_query` to validate assumptions before proposing anything in later steps.\n\n| What to validate | Query to run |\n|-----------------|-------------|\n| **Denormalized vs joined values** | Compare pre-computed columns (e.g., `customers.order_count`) against the actual joined aggregate (`count()` from `orders`). Report discrepancy rate. If >0%, flag for user decision. |\n| **Candidate date fields** | When multiple date/timestamp columns exist, query both. What % of rows differ? By how much? This informs which is canonical. |\n| **Numeric column distributions** | Query min, max, avg, percentiles (p25, p50, p75, p95). These inform tier boundaries and detect outliers. |\n| **Categorical column cardinality** | Query distinct values. A `status` column with 5 values behaves differently from one with 500. |\n| **Column usefulness** | Query NULL rates. Columns that are >95% NULL are candidates for `internal`. |\n| **Join cardinality** | Query FK uniqueness: `group_by: fk_col, aggregate: row_count is count(), having: row_count > 1`. Determines `join_one` vs `join_many`. |\n| **Revenue/amount columns** | When multiple money columns exist (`total`, `subtotal`, `amount`, `price`), query a sample to understand how they relate (does `total = subtotal + tax`?). |\n| **Join key value compatibility** | For every proposed join, sample 5-10 actual values from each side. Check for format mismatches: abbreviations (\"4th Av\" vs \"4 Avenue\"), ordinals (\"23 St\" vs \"23rd St\"), casing, prefixes. Mismatched values mean the join won't work even if column names match. |\n| **Mixed-grain rows** | For each key table, run top-N and bottom-N by primary metric. Look for summary/aggregate rows mixed with detail data (e.g., \"System Total\" rows in a station-level table). These corrupt measures if not filtered out. |\n\n**Never assume from column names.** Always query the data to confirm. A column named `total` could include or exclude tax. A `status` column could have unexpected values. A FK could have orphaned references.\n\n### Example Queries\n\n**Tier boundaries**: query distribution, propose breaks from percentiles:\n```malloy\nrun: orders -> {\n  aggregate:\n    min_val is min(sale_price), p25 is sale_price.percentile(25)\n    median_val is sale_price.percentile(50), p75 is sale_price.percentile(75)\n    p95 is sale_price.percentile(95), max_val is max(sale_price)\n}\n```\n\n**Denormalized vs joined**: compare pre-computed column against real aggregate, report match rate:\n```malloy\nrun: customers -> {\n  join_many: orders on customer_id = orders.customer_id\n  aggregate:\n    total is count()\n    match is count() { where: order_count = count(orders.order_id) }\n}\n```\n\n**Canonical date**: when multiple date columns exist, check how often they differ:\n```malloy\nrun: orders -> {\n  aggregate:\n    total is count()\n    same_date is count() { where: created_at::date = submitted_at::date }\n    max_gap_days is max(days(submitted_at - created_at))\n}\n```\n\n**Revenue columns**: when multiple money columns exist, verify their relationship:\n```malloy\nrun: orders -> {\n  aggregate:\n    total_eq_parts is count() { where: abs(sale_price - (subtotal + tax)) < 0.01 }\n    total is count()\n}\n```\n\n### Schema Shape\n- Is this a star/snowflake schema (use base + joined source layers) or normalized/ER-style (may need 3-stage pattern)?\n- Combined vs split tables: prefer filtered/split tables over combined when both exist.\n\n## Computed Source Detection\n\nFlag potential computed sources when:\n\n1. **Grain mismatch**: the analytical scope requires a grain that no physical table provides (e.g., customer-level metrics from an order-grain table)\n2. **Repeated aggregation patterns**: the same GROUP BY + aggregate pattern would be needed in multiple analyses\n3. **Cross-entity aggregations**: the model or the data implies cross-entity aggregations that require a pre-aggregated entity\n\n## Prior Art Detection\n\nCheck for prior art signals at the start of discovery. If a signal is found and the user confirms, **you MUST read** the corresponding reference skill and follow its instructions.\n\n| Signal | Source Type | Reference to Read |\n|--------|------------|-------------------|\n| `.lkml` files in project or subdirectories | lookml | `skill:malloy-lookml-review` |\n| `dbt_project.yml` in project or parent dirs | dbt | dbt review (future) |\n\nThe reference handles inventory, classification, and produces prior-art notes. Keep those notes in-conversation, then continue with normal discovery below.\n\n**If DB connection available (LookML + DB mode):**\n- Read the model and run `execute_query` as normal\n- Use prior art as additional context, not a replacement for data validation\n- **The LookML connection name is NOT the Malloy connection name.** Always use the connection name from the model.\n\n**If no DB connection (LookML-only mode):**\n- Skip the model-read and `execute_query` steps\n- Use connection name and table paths extracted from prior art source files\n- Flag all proposals in Steps 2-4 as **unvalidated**\n- Proceed directly to Step 2 (PROPOSE SCOPE)\n\n**Prior art findings enhance discovery, they don't replace it.** When a DB connection is available, always validate assumptions against the actual data.\n\n## After Discovery\n\nDo NOT present findings to the user yet.\n\n## Done\n\nStep complete. Output: discovery findings (internal: tables, columns, relationships, data quality, prior art). Continue to the next modeling step (see your modeling workflow).\n\n## Verify Source Joins\n\nWhen reading joins off the model or the data, watch for `join_many` where the actual relationship is many-to-one. Always verify cardinality. Prefer `join_one` when each row in the primary table matches at most one row in the joined table." }, { name: "malloy-document", description: 'Add documentation with #(doc) tags to Malloy models so fields and sources are described in plain language. Use when user asks to "add documentation", "add doc tags", "document the model", or wants fields and sources described for natural-language search and discovery. For declaring parameterizable filters with #(filter), see the malloy-model skill. Filters are a runtime/modeling construct (governance, latency, correctness), not a documentation tag.', body: "# Documenting a Malloy Model\n\nAdd `#(doc)` tags to describe sources and fields in plain language so they are easy to find and understand:\n\n| Tag | Purpose | Goes on |\n|-----|---------|---------|\n| `#(doc)` | Plain-language description for natural-language search | source, dimension, measure, view, join |\n| `#(filter)` | Declare a parameterizable filter (runtime/modeling concern, see `malloy-model`) | source |\n\n`#(doc)` is a standard Malloy annotation. It documents a field or source with a human-readable description that downstream tools can surface and search against.\n\n## #(doc) Tag\n\nAdd before any source, dimension, measure, view, or join. When multiple fields share a keyword, use it once as a block header. Tags and field names are indented under the keyword; tags go on the line(s) directly above the field they annotate.\n\n**Tag ordering** (when a field has multiple tags): `#(doc)` → render tags (`# currency`, `# label`, etc.) → field name. Separate each field group with a blank line:\n\n```malloy\n#(doc) Customer who placed the order\njoin_one: users with user_id\n\ndimension:\n  #(doc) Date the order was placed (UTC)\n  order_date is created_at::date\n\nmeasure:\n  #(doc) Total revenue from all orders in USD\n  # currency\n  revenue is sum(total)\n```\n\n### Writing Doc Strings for Retrieval\n\nDoc strings power natural-language search: users type plain-English questions and the system matches against your `#(doc)` strings. Write descriptions that match how analysts would search:\n\n- **Include business meaning**, not code mechanics: what it represents, not how it's implemented\n- **Include units** (USD, count, percentage) and valid values for categorical fields\n- **Avoid Malloy jargon**: never use \"filterable\", \"groupable\", \"dimension\", \"measure\", \"aggregation\"\n\n**Good examples:**\n- `#(doc) Total revenue from completed orders in USD` matches \"what was our revenue?\"\n- `#(doc) Customer signup date (UTC)` matches \"when did the customer join?\"\n- `#(doc) Order status: pending, processing, shipped, delivered, cancelled` matches \"what are the order statuses?\"\n\n**Bad examples:**\n- `#(doc) Filterable dimension for order status`: no analyst searches for \"filterable\"\n- `#(doc) Groupable by region`: \"groupable\" is a system concept\n- `#(doc) Aggregation of total sales`: \"aggregation\" doesn't match natural queries\n\n## #(filter): see `malloy-model`\n\n`#(filter)` is also a `#(...)`-shaped annotation, but unlike `#(doc)` it's a **runtime/modeling construct**: it shapes governance, query latency, and correctness, not discoverability. The full reference (syntax, filter types, `required` / `implicit` flags, and when each applies) lives in `malloy-model` § Parameterizable Filters with `#(filter)` alongside the other source-authoring constructs.\n\nOne rule worth knowing here: filters live on the source, never on the consumer. Ad-hoc reports and notebooks that import a source inherit its filters automatically; they do not (and cannot) declare new ones.\n\n## `internal:` and `private:`: column-level access in a source\n\n`#(doc)` describes what's exposed. Two access modifiers control what's exposed in the first place, and both live **inside** a source's `include {}` block. They are about the source's public API and data sensitivity, not about documentation, so reach for them when curating which columns callers can pick.\n\n| Mechanism | Layer | Why you reach for it |\n|---|---|---|\n| `internal:` | Inside a source (one column in `include {}`) | The column **isn't part of your model's public API**. Common reasons: data is messy (empty/garbage, raw JSON, duplicates), or a documented derived dimension already supersedes it, or the raw column exists only to be joined on / referenced internally and shouldn't appear as a dimension callers can pick. The data may be perfectly fine, it's just not what you want exposed. |\n| `private:` | Inside a source (one column in `include {}`) | The **data is sensitive**: SSN, raw credit card, password. Governance / security concern; a harder block than `internal:`. |\n\nIn one sentence: **`internal:` and `private:` shape what's inside a source's public API; `#(doc)` describes the fields you do expose.**\n\n### Example\n\nA base source pulled from a messy raw table often uses `internal:` to drop raw fields from the public API, while documenting the curated columns with `#(doc)`.\n\n```malloy\n// orders_base.malloy\n#(doc) Raw orders. Use orders.malloy as the entry point for analysis.\nsource: orders_base is conn.table('orders_raw')\n  include {\n    public: id, customer_id, order_date, total\n    internal: raw_json_payload, deprecated_status_code, _temp_dedup_marker\n  }\n  extend {\n    primary_key: id\n  }\n```\n\n```malloy\n// orders.malloy\nimport \"orders_base.malloy\"\n\n#(doc) Order analysis. Use for revenue, fulfillment, and customer-order joins.\nsource: orders is orders_base extend {\n  // joins, measures, curated dimensions\n}\n```\n\nThe base source stays fully queryable (`run: orders_base -> { ... }` still works); `internal:` only governs which columns appear as public dimensions callers can pick.\n\n## Annotating Columns in Include (Experimental)\n\nWith `##! experimental.access_modifiers`, you can add `#(doc)` tags to raw table columns inside `include` blocks. This documents columns without redefining them as dimensions.\n\n```malloy\n##! experimental.access_modifiers\n\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order line item identifier\n    id\n\n    #(doc) Customer email address\n    email\n\n    #(doc) Order status: pending, shipped, delivered\n    status\n\n  // internal: only for verified noise (empty cols, raw JSON blobs, duplicates)\n}\nextend {\n  // ... dimensions and measures\n}\n```\n\n**When to use:**\n- Documenting raw columns without creating explicit dimensions\n- Curating which columns are public vs internal\n\n## Source-Level Documentation\n\nDocument **when to use** a source, not what it contains. Dimensions and measures can already be searched directly, so the source-level `#(doc)` should describe what questions/analyses this source answers.\n\n**Base source files:** Document what the table represents.\n```malloy\n#(doc) Customer records with demographics and segmentation. One row per customer.\nsource: customers is conn.table('sales.customers') extend { ... }\n```\n\n**Source files:** Document what analytical questions the source answers.\n```malloy\n#(doc) Customer health analysis. Use for retention, segmentation, churn risk, and lifetime value. For order-level analysis, use order_analysis instead.\nsource: customer_health is customers extend { ... }\n```\n\n**Best practices:**\n- Add `#(doc)` to all base source and joined source definitions\n- Base source docs: describe what the table is (one row per what)\n- Source docs: describe what questions/analyses the source answers\n- Documentation happens per-source-file, not in one monolithic file\n\n## Flag Ambiguous Descriptions\n\nAfter writing `#(doc)` tags, present any that required judgment to the user for confirmation:\n\n| Field | Proposed doc | Confidence | Uncertainty |\n|-------|-------------|------------|-------------|\n| `total` | \"Total order amount in USD\" | Medium | Could be gross or net, verified with sample query |\n| `status` | \"Order status: pending, shipped, delivered\" | High | Values confirmed via a query of distinct values |\n\nOnly flag fields where the description required assumptions about business meaning, units, or valid values. When in doubt about valid values, run a quick query against the data to confirm them before writing the description. Use `malloy_getContext` to ground yourself in the package's sources and fields and `malloy_executeQuery` to check distinct values, for example `run: source -> { group_by: status }`.\n\n## Done\n\nStep complete. Output: `#(doc)` tags added to all public fields and sources." }, { name: "malloy-getting-started", description: "First steps for using a Malloy Publisher deployment through its MCP tools. Use when connecting to Publisher for the first time, when you do not yet know the available environments, packages, or models, or when a user asks what data they can explore. Covers verifying the server, discovering data with malloy_getContext, and running a first grounded query.", body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.' }, { name: "malloy-gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: Only `seconds` / `minutes` / `hours` / `days`\n\n`weeks()`, `months()`, `quarters()`, `years()` are **documented but don't work** in this build; only `seconds`, `minutes`, `hours`, `days` actually function. Compute in days and derive the larger unit: a *units conversion*, not a calendar-floored duration:\n\n```malloy\n// WRONG: weeks()/months() don't compile\ndimension: weeks_open is weeks(opened_at to closed_at)\n\n// RIGHT: measure in days, convert (documents that it's approximate)\ndimension: days_open  is days(opened_at to closed_at)\ndimension: weeks_open is days(opened_at to closed_at) / 7      // ≈ weeks\ndimension: months_open is days(opened_at to closed_at) / 30.44 // ≈ months\n```\n\n(Contrast: `search_malloy_docs` gets this right when asked narrowly; trust the docs on the supported units, not on the missing ones.)\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `seconds()`): only `seconds`/`minutes`/`hours`/`days` exist (see above)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "malloy-gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n//   \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`\n\n## Fields Within a Clause: Commas or Newlines, Never Semicolons\n\nSemicolons are not a separator anywhere in Malloy. Multiple fields under one `aggregate:` / `group_by:` are separated by commas (inline) or newlines (one per line); a `;` fails with `no viable alternative at input '<next-field>'` pointing at the field right after it.\n\n```malloy\n// WRONG: semicolons between fields\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT: commas inline...\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n// ...or newlines\nrun: schools -> {\n  aggregate:\n    total is count()\n    charters is count() { where: is_charter }\n}\n```" }, { name: "malloy-gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
+A confirmed source architecture and a confirmed set of field definitions (renames, dimensions, measures, business decisions), held in the conversation and ready to write into the \`.malloy\` model via your modeling workflow.` }, { name: "malloy-discover", description: "Silent data discovery for Malloy modeling. Used at Step 1 of the modeling workflow. Scans tables, columns, distributions, and relationships without user interaction. The agent builds an internal picture before presenting anything.", body: "# Data Discovery (Step 1, Silent)\n\n> **CRITICAL**: Read the model before writing ANY Malloy code. The model defines the sources, connection names, and fields. Never guess connection names.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n> **PREREQUISITE:** Make sure the Malloy MCP tools (`get_context`, `execute_query`, `search_malloy_docs`) are configured and reachable. If they are not, stop and resolve the MCP connection before continuing.\n\n**This step is silent.** The agent does not present findings to the user yet. That happens in the next step (PROPOSE SCOPE).\n\n## Tools\n\n- **`get_context`**: Ground yourself in the package's sources, views, and fields (with their docs). Call FIRST. The sources and their join paths are the schema you build on.\n- **`execute_query`**: Run ad-hoc queries to preview data, verify values, check NULLs, validate assumptions.\n- **`search_malloy_docs`**: Get Malloy syntax help when needed.\n\n## Workflow\n\n```\n1. Check for prior art signals                 → If found, ask user: \"I found [LookML/dbt] files, use as prior art?\"\n2. If user confirms: read adapter reference        → Follow skill:malloy-lookml-review, keep prior-art notes in-conversation\n3. get_context                           → Ground yourself: sources, views, fields\n4. Inspect source definitions                  → See ALL fields and join paths for key sources\n5. Derive candidate joins/dimensions/measures  → Read them off the model and the data, not a suggestion tool\n6. Define a minimal source if one is missing   → Just enough to run execute_query for previews\n7. execute_query(query)                  → Preview data, verify values, check NULLs, check duplicates\n8. search_malloy_docs(query)                    → Get syntax help when needed\n9. Proceed to Step 2 (PROPOSE SCOPE)\n```\n\n**If the model has no sources defined** and no LookML files are present, do NOT silently retry or proceed without data. Tell the user: \"No model sources were found. Please check that the package points at a connected data source, then try again.\"\n\n**If the model has no sources defined** but LookML files ARE present (LookML-only mode), skip steps 3-7. Use connection name and table paths from the LookML review. Flag all proposals as unvalidated.\n\n**Key principle:** Query data to verify assumptions. Don't ask the user to confirm values you can check yourself.\n\n**Search docs proactively.** If you discover patterns that need derived/pre-aggregated sources, window functions, or unfamiliar features, call `search_malloy_docs` BEFORE writing code, not just when you hit errors.\n\n## Query File for Discovery\n\n**In the schema-first workflow:** Run ad-hoc queries with `execute_query`. If the source you want to preview is not yet defined in the model, define a minimal one against the connection and table so you can run previews. The real model fields are built in later steps.\n\n```malloy\n// minimal source for previewing data during discovery\nsource: explore is my_conn.table('schema.table') extend {}\n```\n\n**In analysis-first mode:** There is no temp file. The analysis `.malloy` file IS your working file. It grows throughout the session and becomes the input for formalizing into a model. See `skill:malloy-analyze` for that workflow.\n\n## What to Capture\n\nWhen reviewing tables and columns, capture:\n\n### Table-Level\n- All tables with row counts\n- Connection name and schema (CRITICAL, never guess)\n- Table roles: fact, dimension, bridge, lookup, staging, operational\n- Join relationships (FK → PK mappings)\n\n### Column-Level\n- Primary key and foreign key columns\n- Data types (watch for string dates, arrays, JSON)\n- Reserved word columns that need backticking (`Date`, `Type`, `number`, `source`, etc.)\n- Column cardinality and NULL rates (via `execute_query`)\n- Data distributions for key numeric and categorical columns\n\n### Data Quality\n- **Check for duplicate rows** on primary keys. Run `group_by: pk, aggregate: count(), having: count() > 1` on each key table. Duplicates cause `sum()` to return nonsensical values.\n- **Denormalized count columns**: beware pre-aggregated fields (e.g., `order_count` in a customer table) that may conflict with joined counts.\n- **Delimited list columns**: flag string columns containing comma-separated values.\n\n### Data-Driven Validation\n\n**Every recommendation must be grounded in queried data, not schema inference.** During discovery, run `execute_query` to validate assumptions before proposing anything in later steps.\n\n| What to validate | Query to run |\n|-----------------|-------------|\n| **Denormalized vs joined values** | Compare pre-computed columns (e.g., `customers.order_count`) against the actual joined aggregate (`count()` from `orders`). Report discrepancy rate. If >0%, flag for user decision. |\n| **Candidate date fields** | When multiple date/timestamp columns exist, query both. What % of rows differ? By how much? This informs which is canonical. |\n| **Numeric column distributions** | Query min, max, avg, percentiles (p25, p50, p75, p95). These inform tier boundaries and detect outliers. |\n| **Categorical column cardinality** | Query distinct values. A `status` column with 5 values behaves differently from one with 500. |\n| **Column usefulness** | Query NULL rates. Columns that are >95% NULL are candidates for `internal`. |\n| **Join cardinality** | Query FK uniqueness: `group_by: fk_col, aggregate: row_count is count(), having: row_count > 1`. Determines `join_one` vs `join_many`. |\n| **Revenue/amount columns** | When multiple money columns exist (`total`, `subtotal`, `amount`, `price`), query a sample to understand how they relate (does `total = subtotal + tax`?). |\n| **Join key value compatibility** | For every proposed join, sample 5-10 actual values from each side. Check for format mismatches: abbreviations (\"4th Av\" vs \"4 Avenue\"), ordinals (\"23 St\" vs \"23rd St\"), casing, prefixes. Mismatched values mean the join won't work even if column names match. |\n| **Mixed-grain rows** | For each key table, run top-N and bottom-N by primary metric. Look for summary/aggregate rows mixed with detail data (e.g., \"System Total\" rows in a station-level table). These corrupt measures if not filtered out. |\n\n**Never assume from column names.** Always query the data to confirm. A column named `total` could include or exclude tax. A `status` column could have unexpected values. A FK could have orphaned references.\n\n### Example Queries\n\n**Tier boundaries**: query distribution, propose breaks from percentiles:\n```malloy\nrun: orders -> {\n  aggregate:\n    min_val is min(sale_price), p25 is sale_price.percentile(25)\n    median_val is sale_price.percentile(50), p75 is sale_price.percentile(75)\n    p95 is sale_price.percentile(95), max_val is max(sale_price)\n}\n```\n\n**Denormalized vs joined**: compare pre-computed column against real aggregate, report match rate:\n```malloy\nrun: customers -> {\n  join_many: orders on customer_id = orders.customer_id\n  aggregate:\n    total is count()\n    match is count() { where: order_count = count(orders.order_id) }\n}\n```\n\n**Canonical date**: when multiple date columns exist, check how often they differ:\n```malloy\nrun: orders -> {\n  aggregate:\n    total is count()\n    same_date is count() { where: created_at::date = submitted_at::date }\n    max_gap_days is max(days(submitted_at - created_at))\n}\n```\n\n**Revenue columns**: when multiple money columns exist, verify their relationship:\n```malloy\nrun: orders -> {\n  aggregate:\n    total_eq_parts is count() { where: abs(sale_price - (subtotal + tax)) < 0.01 }\n    total is count()\n}\n```\n\n### Schema Shape\n- Is this a star/snowflake schema (use base + joined source layers) or normalized/ER-style (may need 3-stage pattern)?\n- Combined vs split tables: prefer filtered/split tables over combined when both exist.\n\n## Computed Source Detection\n\nFlag potential computed sources when:\n\n1. **Grain mismatch**: the analytical scope requires a grain that no physical table provides (e.g., customer-level metrics from an order-grain table)\n2. **Repeated aggregation patterns**: the same GROUP BY + aggregate pattern would be needed in multiple analyses\n3. **Cross-entity aggregations**: the model or the data implies cross-entity aggregations that require a pre-aggregated entity\n\n## Prior Art Detection\n\nCheck for prior art signals at the start of discovery. If a signal is found and the user confirms, **you MUST read** the corresponding reference skill and follow its instructions.\n\n| Signal | Source Type | Reference to Read |\n|--------|------------|-------------------|\n| `.lkml` files in project or subdirectories | lookml | `skill:malloy-lookml-review` |\n| `dbt_project.yml` in project or parent dirs | dbt | dbt review (future) |\n\nThe reference handles inventory, classification, and produces prior-art notes. Keep those notes in-conversation, then continue with normal discovery below.\n\n**If DB connection available (LookML + DB mode):**\n- Read the model and run `execute_query` as normal\n- Use prior art as additional context, not a replacement for data validation\n- **The LookML connection name is NOT the Malloy connection name.** Always use the connection name from the model.\n\n**If no DB connection (LookML-only mode):**\n- Skip the model-read and `execute_query` steps\n- Use connection name and table paths extracted from prior art source files\n- Flag all proposals in Steps 2-4 as **unvalidated**\n- Proceed directly to Step 2 (PROPOSE SCOPE)\n\n**Prior art findings enhance discovery, they don't replace it.** When a DB connection is available, always validate assumptions against the actual data.\n\n## After Discovery\n\nDo NOT present findings to the user yet.\n\n## Done\n\nStep complete. Output: discovery findings (internal: tables, columns, relationships, data quality, prior art). Continue to the next modeling step (see your modeling workflow).\n\n## Verify Source Joins\n\nWhen reading joins off the model or the data, watch for `join_many` where the actual relationship is many-to-one. Always verify cardinality. Prefer `join_one` when each row in the primary table matches at most one row in the joined table." }, { name: "malloy-document", description: 'Add documentation with #(doc) tags to Malloy models so fields and sources are described in plain language. Use when user asks to "add documentation", "add doc tags", "document the model", or wants fields and sources described for natural-language search and discovery. For declaring parameterizable filters with #(filter), see the malloy-model skill. Filters are a runtime/modeling construct (governance, latency, correctness), not a documentation tag.', body: "# Documenting a Malloy Model\n\nAdd `#(doc)` tags to describe sources and fields in plain language so they are easy to find and understand:\n\n| Tag | Purpose | Goes on |\n|-----|---------|---------|\n| `#(doc)` | Plain-language description for natural-language search | source, dimension, measure, view, join |\n| `#(filter)` | Declare a parameterizable filter (runtime/modeling concern, see `malloy-model`) | source |\n\n`#(doc)` is a standard Malloy annotation. It documents a field or source with a human-readable description that downstream tools can surface and search against.\n\n## #(doc) Tag\n\nAdd before any source, dimension, measure, view, or join. When multiple fields share a keyword, use it once as a block header. Tags and field names are indented under the keyword; tags go on the line(s) directly above the field they annotate.\n\n**Tag ordering** (when a field has multiple tags): `#(doc)` → render tags (`# currency`, `# label`, etc.) → field name. Separate each field group with a blank line:\n\n```malloy\n#(doc) Customer who placed the order\njoin_one: users with user_id\n\ndimension:\n  #(doc) Date the order was placed (UTC)\n  order_date is created_at::date\n\nmeasure:\n  #(doc) Total revenue from all orders in USD\n  # currency\n  revenue is sum(total)\n```\n\n### Writing Doc Strings for Retrieval\n\nDoc strings power natural-language search: users type plain-English questions and the system matches against your `#(doc)` strings. Write descriptions that match how analysts would search:\n\n- **Include business meaning**, not code mechanics: what it represents, not how it's implemented\n- **Include units** (USD, count, percentage) and valid values for categorical fields\n- **Avoid Malloy jargon**: never use \"filterable\", \"groupable\", \"dimension\", \"measure\", \"aggregation\"\n\n**Good examples:**\n- `#(doc) Total revenue from completed orders in USD` matches \"what was our revenue?\"\n- `#(doc) Customer signup date (UTC)` matches \"when did the customer join?\"\n- `#(doc) Order status: pending, processing, shipped, delivered, cancelled` matches \"what are the order statuses?\"\n\n**Bad examples:**\n- `#(doc) Filterable dimension for order status`: no analyst searches for \"filterable\"\n- `#(doc) Groupable by region`: \"groupable\" is a system concept\n- `#(doc) Aggregation of total sales`: \"aggregation\" doesn't match natural queries\n\n## #(filter): see `malloy-model`\n\n`#(filter)` is also a `#(...)`-shaped annotation, but unlike `#(doc)` it's a **runtime/modeling construct**: it shapes governance, query latency, and correctness, not discoverability. The full reference (syntax, filter types, `required` / `implicit` flags, and when each applies) lives in `malloy-model` § Parameterizable Filters with `#(filter)` alongside the other source-authoring constructs.\n\nOne rule worth knowing here: filters live on the source, never on the consumer. Ad-hoc reports and notebooks that import a source inherit its filters automatically; they do not (and cannot) declare new ones.\n\n## `internal:` and `private:`: column-level access in a source\n\n`#(doc)` describes what's exposed. Two access modifiers control what's exposed in the first place, and both live **inside** a source's `include {}` block. They are about the source's public API and data sensitivity, not about documentation, so reach for them when curating which columns callers can pick.\n\n| Mechanism | Layer | Why you reach for it |\n|---|---|---|\n| `internal:` | Inside a source (one column in `include {}`) | The column **isn't part of your model's public API**. Common reasons: data is messy (empty/garbage, raw JSON, duplicates), or a documented derived dimension already supersedes it, or the raw column exists only to be joined on / referenced internally and shouldn't appear as a dimension callers can pick. The data may be perfectly fine, it's just not what you want exposed. |\n| `private:` | Inside a source (one column in `include {}`) | The **data is sensitive**: SSN, raw credit card, password. Governance / security concern; a harder block than `internal:`. |\n\nIn one sentence: **`internal:` and `private:` shape what's inside a source's public API; `#(doc)` describes the fields you do expose.**\n\n### Example\n\nA base source pulled from a messy raw table often uses `internal:` to drop raw fields from the public API, while documenting the curated columns with `#(doc)`.\n\n```malloy\n// orders_base.malloy\n#(doc) Raw orders. Use orders.malloy as the entry point for analysis.\nsource: orders_base is conn.table('orders_raw')\n  include {\n    public: id, customer_id, order_date, total\n    internal: raw_json_payload, deprecated_status_code, _temp_dedup_marker\n  }\n  extend {\n    primary_key: id\n  }\n```\n\n```malloy\n// orders.malloy\nimport \"orders_base.malloy\"\n\n#(doc) Order analysis. Use for revenue, fulfillment, and customer-order joins.\nsource: orders is orders_base extend {\n  // joins, measures, curated dimensions\n}\n```\n\nThe base source stays fully queryable (`run: orders_base -> { ... }` still works); `internal:` only governs which columns appear as public dimensions callers can pick.\n\n## Annotating Columns in Include (Experimental)\n\nWith `##! experimental.access_modifiers`, you can add `#(doc)` tags to raw table columns inside `include` blocks. This documents columns without redefining them as dimensions.\n\n```malloy\n##! experimental.access_modifiers\n\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order line item identifier\n    id\n\n    #(doc) Customer email address\n    email\n\n    #(doc) Order status: pending, shipped, delivered\n    status\n\n  // internal: only for verified noise (empty cols, raw JSON blobs, duplicates)\n}\nextend {\n  // ... dimensions and measures\n}\n```\n\n**When to use:**\n- Documenting raw columns without creating explicit dimensions\n- Curating which columns are public vs internal\n\n## Source-Level Documentation\n\nDocument **when to use** a source, not what it contains. Dimensions and measures can already be searched directly, so the source-level `#(doc)` should describe what questions/analyses this source answers.\n\n**Base source files:** Document what the table represents.\n```malloy\n#(doc) Customer records with demographics and segmentation. One row per customer.\nsource: customers is conn.table('sales.customers') extend { ... }\n```\n\n**Source files:** Document what analytical questions the source answers.\n```malloy\n#(doc) Customer health analysis. Use for retention, segmentation, churn risk, and lifetime value. For order-level analysis, use order_analysis instead.\nsource: customer_health is customers extend { ... }\n```\n\n**Best practices:**\n- Add `#(doc)` to all base source and joined source definitions\n- Base source docs: describe what the table is (one row per what)\n- Source docs: describe what questions/analyses the source answers\n- Documentation happens per-source-file, not in one monolithic file\n\n## Flag Ambiguous Descriptions\n\nAfter writing `#(doc)` tags, present any that required judgment to the user for confirmation:\n\n| Field | Proposed doc | Confidence | Uncertainty |\n|-------|-------------|------------|-------------|\n| `total` | \"Total order amount in USD\" | Medium | Could be gross or net, verified with sample query |\n| `status` | \"Order status: pending, shipped, delivered\" | High | Values confirmed via a query of distinct values |\n\nOnly flag fields where the description required assumptions about business meaning, units, or valid values. When in doubt about valid values, run a quick query against the data to confirm them before writing the description. Use `malloy_getContext` to ground yourself in the package's sources and fields and `malloy_executeQuery` to check distinct values, for example `run: source -> { group_by: status }`.\n\n## Done\n\nStep complete. Output: `#(doc)` tags added to all public fields and sources." }, { name: "malloy-getting-started", description: "First steps for using a Malloy Publisher deployment through its MCP tools. Use when connecting to Publisher for the first time, when you do not yet know the available environments, packages, or models, or when a user asks what data they can explore. Covers verifying the server, discovering data with malloy_getContext, and running a first grounded query.", body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\nIf the data you want is in a connected database but not yet in any package, use `malloy_searchDatabaseSchema` instead of `malloy_getContext`: it walks a connection\'s schemas and tables and ranks them against a plain-English description, and hands back the `source:` line to start a model from. It returns names and types only, so to see what a column actually contains run `malloy_executeQuery` against a model in a package that uses the same connection, with an ad-hoc query like `run: my_conn.table(\'sales.orders\') -> { group_by: order_status }`. That tool needs an existing model to run against, so a table you have not modelled yet has none of its own.\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.' }, { name: "malloy-gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: Only `seconds` / `minutes` / `hours` / `days`\n\n`weeks()`, `months()`, `quarters()`, `years()` are **documented but don't work** in this build; only `seconds`, `minutes`, `hours`, `days` actually function. Compute in days and derive the larger unit: a *units conversion*, not a calendar-floored duration:\n\n```malloy\n// WRONG: weeks()/months() don't compile\ndimension: weeks_open is weeks(opened_at to closed_at)\n\n// RIGHT: measure in days, convert (documents that it's approximate)\ndimension: days_open  is days(opened_at to closed_at)\ndimension: weeks_open is days(opened_at to closed_at) / 7      // ≈ weeks\ndimension: months_open is days(opened_at to closed_at) / 30.44 // ≈ months\n```\n\n(Contrast: `search_malloy_docs` gets this right when asked narrowly; trust the docs on the supported units, not on the missing ones.)\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `seconds()`): only `seconds`/`minutes`/`hours`/`days` exist (see above)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "malloy-gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n//   \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`\n\n## Fields Within a Clause: Commas or Newlines, Never Semicolons\n\nSemicolons are not a separator anywhere in Malloy. Multiple fields under one `aggregate:` / `group_by:` are separated by commas (inline) or newlines (one per line); a `;` fails with `no viable alternative at input '<next-field>'` pointing at the field right after it.\n\n```malloy\n// WRONG: semicolons between fields\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT: commas inline...\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n// ...or newlines\nrun: schools -> {\n  aggregate:\n    total is count()\n    charters is count() { where: is_charter }\n}\n```" }, { name: "malloy-gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
 
 > **Read this before adding renderer annotations.** These patterns cause most rendering issues.
 
@@ -289228,7 +289916,7 @@ source: first_touch is conn.table('events') -> {
 - **Cannot redefine** columns from query-based sources, they already exist as fields. Add only NEW derived dimensions in \`extend {}\`.
 - To add \`#(doc)\` tags to existing query columns, use \`include {}\` between the query and extend.
 - **Use the RAW TABLE** in query-based sources, not a modeled source, when the modeled source would create a circular dependency.
-- **Never use \`conn.sql()\`** when Malloy has a native pattern. \`conn.sql()\` is a last resort for UNNEST, PIVOT, or dialect-specific functions only. Call \`search_malloy_docs\` first.` }, { name: "malloy-modeling", description: "Build semantic models with Malloy for the Malloy Publisher. Read this skill whenever the user asks about modeling data or specifically mentions Malloy.", body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself with `malloy_getContext` before writing ANY code. It returns the package's sources, views, and fields (with their docs), so you build on what actually exists. Never guess field names.\n2. **Search docs proactively**: call `malloy_searchDocs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **Use `count(x)` not `count(distinct x)`**: Malloy's count() is always distinct\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files. Once approved, you can write a `modeling-notes.md` during execution to record decisions (scope, sources, key choices, prior art, gaps). This file persists alongside the model. Otherwise, keep the proposal and decisions in the conversation; Publisher has no separate workspace document store to write them to.\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions; load the relevant skill when needed.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags.** Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent)  (user)   (user)      (user)       (agent)      (agent)      (user)   (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present structure, assumptions, and doc coverage; user confirms |\n| 8. Curate | `skill:malloy-model` | Propose access controls, user approves: optional, ask user |\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** \"Explore this data\" → `skill:malloy-analyze` → formalize via `skill:malloy-model` (`reference/analysis-to-model.md`)\n\nAfter analysis completes, **always recommend formalizing into a model.**\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through review. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-analyze` first, then formalize via `skill:malloy-model` |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-analyze`. After analysis completes, **always recommend formalizing via the analysis-to-model workflow** (`skill:malloy-model` → `reference/analysis-to-model.md`).\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling.\n\n| Tool | Purpose |\n|------|---------|\n| `malloy_getContext` | Ground yourself in a package: its sources, views, and fields |\n| `malloy_executeQuery` | Run ad-hoc queries for validation |\n| `malloy_compile` | Compile-check a change and get diagnostics back without running a query |\n| `malloy_reloadPackage` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `malloy_searchDocs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n\nNever guess field names. Ground yourself with `malloy_getContext` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `malloy_compile`, which reads the model fresh from disk and returns diagnostics without running anything.\n2. **Save** it to the package's model file.\n3. **Reload** with `malloy_reloadPackage`.\n4. **Run** the new view with `malloy_executeQuery`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these two tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `malloy_executeQuery`.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ...           RIGHT: source: flights is ...\nWRONG: dimension: x as y               RIGHT: dimension: y is x\nWRONG: count(*)                        RIGHT: count()\nWRONG: count(distinct x)               RIGHT: count(x)\nWRONG: revenue / order_count           RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... }                RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns" }, { name: "malloy-notebook-chat", description: "Steps to follow when the chat is bound to a notebook or saved report. The notebook's cells are the agent's primary context, answer from it, run its queries, and only reach for get_context when the user asks about something outside it.", body: `# Notebook/Report Chat Workflow
+- **Never use \`conn.sql()\`** when Malloy has a native pattern. \`conn.sql()\` is a last resort for UNNEST, PIVOT, or dialect-specific functions only. Call \`search_malloy_docs\` first.` }, { name: "malloy-modeling", description: "Build semantic models with Malloy for the Malloy Publisher. Read this skill whenever the user asks about modeling data or specifically mentions Malloy.", body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself before writing ANY code, with the tool that matches what you are modelling.\n   - Modelling data **already in a package**: `malloy_getContext` returns that package's sources, views, and fields (with their docs).\n   - Modelling **a database with no package yet**: `malloy_getContext` has nothing to return, so use `malloy_searchDatabaseSchema` instead. It walks the connection's schemas and tables, ranks them against a plain-English description, and gives you each table's columns plus the `source:` line to start from. Take those names verbatim into step 5.\n   Never guess field names either way.\n2. **Search docs proactively**: call `malloy_searchDocs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **Use `count(x)` not `count(distinct x)`**: Malloy's count() is always distinct\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files. Once approved, you can write a `modeling-notes.md` during execution to record decisions (scope, sources, key choices, prior art, gaps). This file persists alongside the model. Otherwise, keep the proposal and decisions in the conversation; Publisher has no separate workspace document store to write them to.\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions; load the relevant skill when needed.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags.** Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent)  (user)   (user)      (user)       (agent)      (agent)      (user)   (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art. With no package yet, start from `malloy_searchDatabaseSchema` to find the tables in the connection |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present structure, assumptions, and doc coverage; user confirms |\n| 8. Curate | `skill:malloy-model` | Propose access controls, user approves: optional, ask user |\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** \"Explore this data\" → `skill:malloy-analyze` → formalize via `skill:malloy-model` (`reference/analysis-to-model.md`)\n\nAfter analysis completes, **always recommend formalizing into a model.**\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through review. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-analyze` first, then formalize via `skill:malloy-model` |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-analyze`. After analysis completes, **always recommend formalizing via the analysis-to-model workflow** (`skill:malloy-model` → `reference/analysis-to-model.md`).\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling.\n\n| Tool | Purpose |\n|------|---------|\n| `malloy_getContext` | Ground yourself in a package: its sources, views, and fields |\n| `malloy_executeQuery` | Run ad-hoc queries for validation |\n| `malloy_compile` | Compile-check a change and get diagnostics back without running a query |\n| `malloy_reloadPackage` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `malloy_searchDocs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n| `malloy_searchDatabaseSchema` | Find the tables in a database connection by plain-English description, when modelling data that is not in a package yet. Returns each table's columns and the `source:` line to start from. Names and types only: no row value is returned |\n\nNever guess field names. Ground yourself with `malloy_getContext` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `malloy_compile`, which reads the model fresh from disk and returns diagnostics without running anything.\n2. **Save** it to the package's model file.\n3. **Reload** with `malloy_reloadPackage`.\n4. **Run** the new view with `malloy_executeQuery`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these two tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `malloy_executeQuery`.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ...           RIGHT: source: flights is ...\nWRONG: dimension: x as y               RIGHT: dimension: y is x\nWRONG: count(*)                        RIGHT: count()\nWRONG: count(distinct x)               RIGHT: count(x)\nWRONG: revenue / order_count           RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... }                RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns" }, { name: "malloy-notebook-chat", description: "Steps to follow when the chat is bound to a notebook or saved report. The notebook's cells are the agent's primary context, answer from it, run its queries, and only reach for get_context when the user asks about something outside it.", body: `# Notebook/Report Chat Workflow
 
 Steps to follow when the user asks a question:
 
@@ -290245,6 +290933,8 @@ var MCP_INSTRUCTIONS = `Malloy Publisher serves one or more Malloy semantic-mode
 
 Start with malloy_getContext. Call it with no arguments to list the environments (each with its packages), with an environment to list its packages, with a package to list its sources, and with a package plus a plain-English question to get the sources, views, and fields most relevant to it. Use the names it returns verbatim and do not guess. Then run a query with malloy_executeQuery. To change a model: validate the edit with malloy_compile, save it, then call malloy_reloadPackage so the new sources and views become queryable by name without restarting the server. ${RELOAD_FAILURE_IS_SAFE}
 
+To build a model from a database rather than from an existing package, start with malloy_searchDatabaseSchema: it lists the connections, their schemas, and their tables, and ranks those tables against a plain-English description of the data you want. Each table it returns carries the source line to start from. It returns names and types only: no row value is returned.
+
 Task-specific guidance is served as prompts you can fetch by name: malloy-getting-started to begin, malloy-modeling to build or change a model, malloy-analysis to explore and answer questions, and malloy-review to check correctness.
 
 Results and any charts render in the Publisher web UI on the REST port (4000 by default).`;
@@ -290259,6 +290949,7 @@ function initializeMcpServer(environmentStore) {
   registerDocsSearchTool(mcpServer, environmentStore);
   registerCompileTool(mcpServer, environmentStore);
   registerReloadPackageTool(mcpServer, environmentStore);
+  registerSearchDatabaseSchemaTool(mcpServer, environmentStore);
   for (const skill of AGENT_SKILLS) {
     mcpServer.prompt(skill.name, skill.description, () => ({
       messages: [