@malloy-publisher/server 0.0.224 → 0.0.226

package/dist/server.mjs

@@ -115107,6 +115107,50 @@ function formatDuration(durationMs) {
   }
   return `${durationMs.toFixed(2)}ms`;
 }
+function redactSensitive(value, seen = new WeakSet) {
+  if (value === null || typeof value !== "object" || value instanceof Date) {
+    return value;
+  }
+  if (seen.has(value)) {
+    return "[Circular]";
+  }
+  seen.add(value);
+  if (Array.isArray(value)) {
+    return value.map((item) => redactSensitive(item, seen));
+  }
+  const result = {};
+  for (const [key, val] of Object.entries(value)) {
+    result[key] = SENSITIVE_KEYS.has(key.toLowerCase()) ? REDACTED : redactSensitive(val, seen);
+  }
+  return result;
+}
+function buildAxiosErrorLog(error) {
+  if (error.response) {
+    return {
+      message: "Axios server-side error",
+      meta: {
+        url: error.response.config.url,
+        status: error.response.status,
+        headers: redactSensitive(error.response.headers),
+        data: redactSensitive(error.response.data)
+      }
+    };
+  }
+  if (error.request) {
+    return {
+      message: "Axios client-side error",
+      meta: {
+        method: error.config?.method,
+        url: error.config?.url,
+        code: error.code
+      }
+    };
+  }
+  return {
+    message: "Axios unknown error",
+    meta: { message: error.message }
+  };
+}
 var import_winston, isTelemetryEnabled, VALID_LOG_LEVELS, getLogLevel = () => {
   if (process.env.LOG_LEVEL) {
     const logLevel = process.env.LOG_LEVEL.toLowerCase();
@@ -115117,7 +115161,7 @@ var import_winston, isTelemetryEnabled, VALID_LOG_LEVELS, getLogLevel = () => {
     }
   }
   return "debug";
-}, logger, DISABLE_RESPONSE_LOGGING, loggerMiddleware = (req, res, next) => {
+}, logger, DISABLE_RESPONSE_LOGGING, SENSITIVE_KEY_NAMES, SENSITIVE_KEYS, REDACTED = "[REDACTED]", loggerMiddleware = (req, res, next) => {
   const startTime = performance.now();
   const resJson = res.json;
   res.json = (body) => {
@@ -115132,12 +115176,12 @@ var import_winston, isTelemetryEnabled, VALID_LOG_LEVELS, getLogLevel = () => {
     const logMetadata = {
       statusCode: res.statusCode,
       duration: formatDuration(durationMs),
-      payload: req.body,
+      payload: redactSensitive(req.body),
       params: req.params,
       query: req.query
     };
     if (!DISABLE_RESPONSE_LOGGING) {
-      logMetadata.response = res.locals.body;
+      logMetadata.response = redactSensitive(res.locals.body);
     }
     if (traceId) {
       logMetadata.traceId = traceId;
@@ -115148,18 +115192,8 @@ var import_winston, isTelemetryEnabled, VALID_LOG_LEVELS, getLogLevel = () => {
   });
   next();
 }, logAxiosError = (error) => {
-  if (error.response) {
-    logger.error("Axios server-side error", {
-      url: error.response.config.url,
-      status: error.response.status,
-      headers: error.response.headers,
-      data: error.response.data
-    });
-  } else if (error.request) {
-    logger.error("Axios client-side error", { error: error.request });
-  } else {
-    logger.error("Axios unknown error", { error });
-  }
+  const { message, meta } = buildAxiosErrorLog(error);
+  logger.error(message, meta);
 };
 var init_logger = __esm(() => {
   import_winston = __toESM(require_winston(), 1);
@@ -115171,6 +115205,28 @@ var init_logger = __esm(() => {
     transports: [new import_winston.default.transports.Console]
   });
   DISABLE_RESPONSE_LOGGING = process.env.DISABLE_RESPONSE_LOGGING === "true" || process.env.DISABLE_RESPONSE_LOGGING === "1";
+  SENSITIVE_KEY_NAMES = [
+    "password",
+    "connectionString",
+    "serviceAccountKeyJson",
+    "privateKey",
+    "privateKeyPass",
+    "secret",
+    "secretAccessKey",
+    "sessionToken",
+    "sasUrl",
+    "clientSecret",
+    "oauthClientSecret",
+    "peakaKey",
+    "token",
+    "accessToken",
+    "authorization",
+    "proxy-authorization",
+    "cookie",
+    "set-cookie",
+    "x-api-key"
+  ];
+  SENSITIVE_KEYS = new Set(SENSITIVE_KEY_NAMES.map((name) => name.toLowerCase()));
 });
 
 // ../../node_modules/bytes/index.js
@@ -158407,6 +158463,11 @@ var require_utils74 = __commonJS((exports, module) => {
   };
 });
 
+// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
+var require_sshcrypto = __commonJS((exports, module) => {
+  module.exports = __require("./sshcrypto-8m50vnmb.node");
+});
+
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
 var require_poly1305 = __commonJS((exports, module) => {
   var __dirname = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto", __filename = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto/poly1305.js";
@@ -158893,7 +158954,7 @@ var require_crypto = __commonJS((exports, module) => {
   var ChaChaPolyDecipher;
   var GenericDecipher;
   try {
-    binding = (()=>{throw new Error("Cannot require module "+"./crypto/build/Release/sshcrypto.node");})();
+    binding = require_sshcrypto();
     ({
       AESGCMCipher,
       ChaChaPolyCipher,
@@ -249955,7 +250016,7 @@ init_errors();
 function formatPublishRejections(pkg, exploresOverride) {
   const message = [
     pkg.formatInvalidExplores(exploresOverride),
-    pkg.formatInvalidSchedule()
+    pkg.formatInvalidPersistencePolicy()
   ].filter(Boolean).join(`
 `);
   return message || undefined;
@@ -258421,6 +258482,7 @@ function hydrateMarkdownOnlyCells(notebookCells) {
 }
 
 // src/service/build_plan.ts
+var FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"];
 function deriveColumns(persistSource) {
   try {
     return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
@@ -258447,6 +258509,60 @@ function deriveAnnotationFields(persistSource) {
   } catch {}
   return out;
 }
+function tagFreshnessLayer(tag) {
+  if (!tag || typeof tag.text !== "function")
+    return {};
+  const layer = {};
+  const window2 = tag.text("freshness", "window");
+  if (typeof window2 === "string")
+    layer.window = window2;
+  const fallback = tag.text("freshness", "fallback");
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  return layer;
+}
+function packageFreshnessLayer(cfg) {
+  if (!cfg)
+    return {};
+  const layer = {};
+  if (cfg.freshness?.window)
+    layer.window = cfg.freshness.window;
+  const fallback = cfg.freshness?.fallback;
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  return layer;
+}
+function safeSourceTag(source) {
+  try {
+    return source.annotations.parseAsTag("@").tag;
+  } catch {
+    return;
+  }
+}
+function safeModelTag(source) {
+  try {
+    return source.modelAnnotations.parseAsTag().tag;
+  } catch {
+    return;
+  }
+}
+function resolveFreshness(source, packageMaterialization) {
+  const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
+  const modelLayer = tagFreshnessLayer(safeModelTag(source));
+  const pkgLayer = packageFreshnessLayer(packageMaterialization);
+  const window2 = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
+  const fallback = sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
+  if (window2 === undefined && fallback === undefined)
+    return null;
+  const freshness = {};
+  if (window2 !== undefined)
+    freshness.window = window2;
+  if (fallback !== undefined)
+    freshness.fallback = fallback;
+  return freshness;
+}
 function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
@@ -258539,7 +258655,7 @@ async function compilePackageBuildPlan(pkg, signal) {
     sourceModelPaths
   };
 }
-function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths) {
+function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths, packageMaterialization) {
   const include = sourceNames ? new Set(sourceNames) : null;
   const wireGraphs = graphs.map((graph) => ({
     connectionName: graph.connectionName,
@@ -258560,8 +258676,8 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       dialect: source.dialectName,
       sourceEntityId: computeSourceEntityId(source, connectionDigests),
       sql: source.getSQL(),
-      sharing: annotationFields.sharing ?? null,
       refresh: annotationFields.refresh ?? null,
+      freshness: resolveFreshness(source, packageMaterialization),
       columns: deriveColumns(source),
       annotationFields,
       modelPath: sourceModelPaths?.[sourceID]
@@ -258574,7 +258690,7 @@ async function computePackageBuildPlan(pkg, signal) {
   if (compiled.graphs.length === 0) {
     return null;
   }
-  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths);
+  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths, pkg.getMaterializationConfig?.() ?? null);
 }
 
 // src/service/freshness.ts
@@ -258781,7 +258897,8 @@ class Package {
       materialization: outcome.packageMetadata.materialization ?? {
         schedule: null,
         freshness: null
-      }
+      },
+      scope: outcome.packageMetadata.scope ?? "package"
     };
     const models = new Map;
     const renderTagWarnings = [];
@@ -258835,11 +258952,11 @@ class Package {
         detail: invalidMsg
       });
     }
-    const invalidSchedule = pkg.formatInvalidSchedule();
-    if (invalidSchedule) {
-      logger.warn(`Package ${packageName} has an invalid cron schedule`, {
+    const invalidPolicy = pkg.formatInvalidPersistencePolicy();
+    if (invalidPolicy) {
+      logger.warn(`Package ${packageName} has an invalid persistence policy`, {
         packageName,
-        detail: invalidSchedule
+        detail: invalidPolicy
       });
     }
     pkg.logEmptyDiscoveryWarnings();
@@ -258851,6 +258968,9 @@ class Package {
   getBuildPlan() {
     return this.buildPlan;
   }
+  getMaterializationConfig() {
+    return this.packageMetadata.materialization ?? null;
+  }
   getPackageMetadata() {
     const metadata = {
       ...this.packageMetadata,
@@ -258935,16 +259055,35 @@ class Package {
     return this.exploreWarnings(exploresOverride).join(`
 `);
   }
-  scheduleWarnings() {
-    const schedule = this.packageMetadata.materialization?.schedule;
-    if (!schedule)
-      return [];
-    return [
-      `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is not accepted: ` + `package-level crons are rejected in Phase B. Declare ` + `'materialization.freshness.window' instead (the control plane schedules ` + `refreshes from it).`
-    ];
+  persistencePolicyWarnings() {
+    const warnings = [];
+    const sources = this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [];
+    const materialization = this.packageMetadata.materialization;
+    const packageSchedule = materialization?.schedule ?? null;
+    const packageFreshness = materialization?.freshness ?? null;
+    const scope = this.packageMetadata.scope ?? "package";
+    for (const source of sources) {
+      const fields = source.annotationFields ?? {};
+      if (fields.sharing !== undefined) {
+        warnings.push(`#@ persist source "${source.name}" declares sharing=... which is ` + `no longer supported: scope is a single package-level mode. Set ` + `the root-level "scope": "version" | "package" in ` + `${PACKAGE_MANIFEST_NAME} instead.`);
+      }
+      if (fields.schedule !== undefined) {
+        warnings.push(`#@ persist source "${source.name}" declares schedule=... which is ` + `no longer supported: a schedule is package-root-only. Declare ` + `"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` + `(requires "scope": "version") instead.`);
+      }
+    }
+    if (packageSchedule && scope !== "version") {
+      warnings.push(`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` + `"scope": "version": a package-scoped lineage is reused across ` + `versions, so a single per-version cadence is meaningless. Set ` + `"scope": "version", or remove the schedule.`);
+    }
+    if (packageSchedule) {
+      const freshnessDeclared = !!(packageFreshness && (packageFreshness.window || packageFreshness.fallback)) || sources.some((s) => s.freshness != null);
+      if (freshnessDeclared) {
+        warnings.push(`materialization.schedule and freshness are mutually exclusive in ` + `${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` + `or freshness (objective tier), never both.`);
+      }
+    }
+    return warnings;
   }
-  formatInvalidSchedule() {
-    return this.scheduleWarnings().join(`
+  formatInvalidPersistencePolicy() {
+    return this.persistencePolicyWarnings().join(`
 `);
   }
   emptyDiscoveryWarnings() {
@@ -259783,7 +259922,8 @@ ${source}` : source;
         explores,
         queryableSources,
         manifestLocation,
-        materialization: existing.materialization
+        materialization: existing.materialization,
+        scope: existing.scope
       });
       const invalidMsg = _package.formatInvalidExplores();
       if (invalidMsg) {
@@ -261261,7 +261401,14 @@ class MaterializationController {
   validateCreateBody(body) {
     const result = {};
     if (body.buildInstructions !== undefined && body.buildInstructions !== null) {
-      result.buildInstructions = this.validateBuildInstructions(body.buildInstructions);
+      const parsed = this.validateBuildInstructions(body.buildInstructions);
+      result.buildInstructions = parsed.sources;
+      if (parsed.referenceManifest !== undefined) {
+        result.referenceManifest = parsed.referenceManifest;
+      }
+      if (parsed.strictUpstreams !== undefined) {
+        result.strictUpstreams = parsed.strictUpstreams;
+      }
     }
     if (body.forceRefresh !== undefined) {
       if (typeof body.forceRefresh !== "boolean") {
@@ -261281,11 +261428,42 @@ class MaterializationController {
     if (typeof raw !== "object" || raw === null) {
       throw new BadRequestError("buildInstructions must be an object");
     }
-    const sources = raw.sources;
+    const obj = raw;
+    const sources = obj.sources;
     if (!Array.isArray(sources) || sources.length === 0) {
       throw new BadRequestError("buildInstructions requires a non-empty 'sources' array of BuildInstruction");
     }
-    return sources.map((instruction) => this.validateInstruction(instruction));
+    const result = {
+      sources: sources.map((instruction) => this.validateInstruction(instruction))
+    };
+    if (obj.referenceManifest !== undefined && obj.referenceManifest !== null) {
+      if (!Array.isArray(obj.referenceManifest)) {
+        throw new BadRequestError("buildInstructions.referenceManifest must be an array of ManifestReference");
+      }
+      result.referenceManifest = obj.referenceManifest.map((ref) => this.validateManifestReference(ref));
+    }
+    if (obj.strictUpstreams !== undefined) {
+      if (typeof obj.strictUpstreams !== "boolean") {
+        throw new BadRequestError("buildInstructions.strictUpstreams must be a boolean");
+      }
+      result.strictUpstreams = obj.strictUpstreams;
+    }
+    return result;
+  }
+  validateManifestReference(raw) {
+    if (typeof raw !== "object" || raw === null) {
+      throw new BadRequestError("Each manifest reference must be an object");
+    }
+    const ref = raw;
+    for (const field of ["sourceEntityId", "physicalTableName"]) {
+      if (typeof ref[field] !== "string") {
+        throw new BadRequestError(`Manifest reference is missing required string field '${field}'`);
+      }
+    }
+    return {
+      sourceEntityId: ref.sourceEntityId,
+      physicalTableName: ref.physicalTableName
+    };
   }
   validateInstruction(raw) {
     if (typeof raw !== "object" || raw === null) {
@@ -265788,7 +265966,7 @@ For small targeted changes (fix one cell, insert one new cell), edit that cell i
 
 ## IMPORTANT
 
-You CANNOT see the rendered output of notebook cells. Do not claim to see charts, values, or patterns from report cells you haven't explicitly executed via \`malloy_executeQuery\`. If you need to analyze results, run the query via \`malloy_executeQuery\` first.` }, { name: "gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` (reference/access-modifiers.md).\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `malloy_searchDocs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `malloy_searchDocs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `malloy_searchDocs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions\n- Time interval functions (`days()`, `seconds()`)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`" }, { name: "gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
+You CANNOT see the rendered output of notebook cells. Do not claim to see charts, values, or patterns from report cells you haven't explicitly executed via \`malloy_executeQuery\`. If you need to analyze results, run the query via \`malloy_executeQuery\` first.` }, { name: "gotchas-modeling", description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.", body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## Field Management: `extend {}` vs `include {}` Don't Compose\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` is the fallback when a `rename:` is unavoidable.** They have different capabilities and **do not combine**.\n\n| Mechanism | Where it lives | Keywords | Compatible with `rename:`? | Experimental flag? |\n|---|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | **No** | Yes (`##! experimental.access_modifiers`) |\n| Field management (fallback) | `extend {}` | `accept:` / `except:` / `rename:` | Yes (same block) | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` (reference/access-modifiers.md).\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When `rename:` is unavoidable: fall back to `extend {}`\n\n`include {}` does not compose with `rename:`. The combination errors with `Can't find field 'X' to set access modifier` because `rename:` runs first and leaves no `X` for `include` to attach a modifier to. There's also a collision inside `include {}` itself: a measure cannot share a name with a raw column, even one tagged `internal:` (`Cannot redefine 'X'`), and the natural fix for that is `rename:`, which then triggers the first error.\n\nWhen a rename is genuinely required (most often during `conn.sql()` to `conn.table()` migration where a SQL alias matches a measure name that's already in heavy use downstream), drop `include {}` and curate the source with `extend { except: ... }` + `rename:` instead. You forfeit `#(doc)` on raw columns and the `public/internal/private` tiers, but keep column gating and the rename.\n\n```malloy\n// RIGHT: rename is required to free `revenue` for the measure\nextend {\n  except: legacy_status_code   // hide garbage column without include {}\n  rename: raw_revenue is revenue\n  measure: revenue is raw_revenue.sum()\n}\n```\n\nIf you can rename the measure or split the source instead, prefer that: it preserves `include {}` and the curated surface.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: preferably `include { internal: ... }` (lets you also `#(doc)` the public columns). If a `rename:` is also needed in the same source, fall back to `extend { except: ... }`.\n4. SQL aliases: `extend { rename: ... }` (forces the fallback path, since `rename:` and `include {}` don't compose). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `malloy_searchDocs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `malloy_searchDocs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `malloy_searchDocs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions\n- Time interval functions (`days()`, `seconds()`)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`" }, { name: "gotchas-queries", description: "Common Malloy query and view mistakes. Read BEFORE writing views, queries, or notebooks. Covers chart constraints, aggregate filters, joined field aliasing, method syntax, and time truncation vs extraction.", body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n  group_by: status\n  aggregate: n is count()\n  limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`" }, { name: "gotchas-rendering", description: "Common Malloy renderer annotation mistakes. Read BEFORE adding chart annotations, formatting tags, or building dashboards. Covers tag syntax, scale rules, sparkline setup, and big_value patterns.", body: `# Rendering Gotchas
 
 > **Read this before adding renderer annotations.** These patterns cause most rendering issues.
 
@@ -265875,7 +266053,16 @@ view: rev_delta is {
 }
 \`\`\`
 
-Use \`down_is_good=true\` for metrics where decrease is positive (churn, defects).` }, { name: "html-data-app-embedding", description: "Embed an in-package HTML data app into a host page or another application, including auto-sizing and auth. Read when embedding a Publisher page via Publisher.embed.", body: '# Embedding an HTML Data App\n\n> `Publisher.embed(selector, { src })` drops a package page into a host page as a sandboxed, auto-resizing iframe. Same-origin embeds authenticate with the browser\'s cookies; cross-origin embeds need a signed token.\n\n## The host-page pattern\n\n```html\n<script src="https://your-publisher/sdk/publisher.js"></script>\n<div id="dashboard"></div>\n<script>\n  const handle = Publisher.embed("#dashboard", {\n    src: "https://your-publisher/environments/demo/packages/sales/index.html",\n  });\n  // handle.destroy() removes the iframe and detaches its listeners.\n</script>\n```\n\n`embed(selector, options)` returns `{ iframe, destroy() }`. Options: `src` (required), `token` (a signed token for cross-origin auth, appended as `embed_token`), `height` (omit to auto-size; a number is treated as pixels), and `allow` (the iframe permissions policy).\n\n## Sizing and the resize contract\n\nOmit `height` and the frame auto-sizes. The embedded page measures its real content height and posts a `publisher:resize` message to the host, which resizes the iframe and accepts that message only from the iframe it created. You write none of this; it ships in `/sdk/publisher.js`, so the embedded page only has to load that script.\n\nDo not rely on `body { min-height: 100vh }` to drive the frame height. The runtime deliberately measures the content\'s bottom edge, not the viewport, to avoid a grow-forever loop.\n\n## Auth\n\n- Same-origin or same-tenant: pass no token. The browser\'s cookies authenticate the iframe.\n- Cross-origin: mint a short-lived signed token server-side and pass it as `options.token`. The runtime appends it to the iframe URL as `embed_token`; the embedded page must read it (from `location.search`) and call `Publisher.setToken(token)`. Because it rides in the URL, it can land in browser history, Referer headers, and server logs, so keep it short-lived and scoped to that one embed, and never put a long-lived or admin token in client HTML.\n\n## Guardrails (v1)\n\n- The iframe is sandboxed (`allow-scripts allow-same-origin allow-forms`). Design for that: no top-level navigation, no popups.\n- Embedded author JavaScript runs with the viewing user\'s data authority, so treat everything under `public/` as strictly first-party code: do not load untrusted third-party scripts, and do not move query results off to other hosts. Tighter per-embed isolation is planned.' }, { name: "html-data-app-runtime", description: "Write the JavaScript that drives an in-package HTML data app, calling Publisher.query, building queries from filter state, and handling results and errors. Read before writing the page's data code.", body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"carriers.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName`, `filterParams` (values for the model\'s legacy `#(filter)` source filters; `Publisher.query` does not forward Malloy `given:` values), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese run against the example `carriers` package (source `carriers`; views `by_letter`, `by_size_bucket`, `kpis`). Swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("carriers.malloy", "run: carriers -> by_letter");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n  const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n  const parts = [];\n  if (state.letter) parts.push(`letter = \'${q(state.letter)}\'`);\n  if (state.bucket) parts.push(`size_bucket = \'${q(state.bucket)}\'`);\n  return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n  "carriers.malloy",\n  `run: carriers -> by_letter + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input. A data-app page has no clean server-side parameterization for arbitrary input today: `Publisher.query` forwards `opts.filterParams` (the deprecated `#(filter)` source-filter API), not Malloy `given:` values. So constrain the input to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("carriers.malloy", "run: carriers -> kpis");\nel.textContent = kpis.total;   // the result is an array; kpis.total, not rows.total\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [byLetter, byBucket, kpisRows] = await Promise.all([\n  Publisher.query("carriers.malloy", "run: carriers -> by_letter"),\n  Publisher.query("carriers.malloy", "run: carriers -> by_size_bucket"),\n  Publisher.query("carriers.malloy", "run: carriers -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n  ```js\n  // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n  // date to a "YYYY-MM" string; monthLabel formats it for display.\n  // target may not cover every actual month; an absent month stays null, never 0.\n  const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n  const data = actualRows.map((r) => ({\n    label: monthLabel(r.order_month),\n    actual: Number(r.revenue),\n    target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n  }));\n  ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("carriers.malloy", "run: carriers -> by_letter")\n  .then((rows) => {\n    if (!rows.length) { el.textContent = "No data."; return; }\n    render(rows);\n  })\n  .catch((err) => {\n    el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n  });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("carriers.malloy", "run: carriers -> by_letter");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The shipped example renders rows with a plain chart library instead, so this path is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: letter` gives a `letter` column; `aggregate: n is count()` gives an `n` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"carriers.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400, required parameter | The model marks a runtime parameter (given) as required. Supply it via `opts.filterParams`, or pass `bypassFilters: true` only if you are a trusted caller. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |' }, { name: "html-data-apps", description: "Build or modify an in-package HTML data app for a Malloy Publisher package (a public/ directory the package serves). Use when the user wants a hand-authored HTML dashboard or web page backed by a package's Malloy models, with no build step.", body: `# In-Package HTML Data Apps
+Use \`down_is_good=true\` for metrics where decrease is positive (churn, defects).
+
+## Theming
+
+- **Per-chart theme keys are nested, not flat.** In Publisher, \`# theme.palette.tableHeader.dark = "#94a3b8"\` works; a flat \`# theme.tableHeaderColor = "#94a3b8"\` is silently dropped. Publisher's annotation reader only understands the structured \`palette.*\` / \`font.*\` form (the same vocabulary as the config), not the renderer's flat \`MalloyExplicitTheme\` key names.
+- **A per-chart annotation OVERRIDES the instance theme.** Precedence, highest to lowest, per key: \`# theme.*\` (view), then \`## theme.*\` (model), then the instance theme (config / Settings, then Theme editor), then built-in defaults. A \`# theme.*\` view tag beats a \`## theme.*\` model default, and both beat the instance theme for the keys they set. This is the opposite of a bare \`@malloydata/render\` embed, where the embedder wins.
+- **Only seven palette keys take light/dark.** \`background\`, \`tableHeader\`, \`tableHeaderBackground\`, \`tableBody\`, \`tile\`, \`tileTitle\`, and \`mapColor\` each accept a \`.light\` and/or \`.dark\` variant. \`palette.series\`, \`font.family\`, and \`font.size\` are single values shared across modes; a \`.light\`/\`.dark\` on them does nothing.
+- **\`# theme.palette.mapColor.{light,dark}\` recolors choropleths only.** It sets the saturated end of the \`# shape_map\` / \`# segment_map\` gradient (per mode). Rect-mark heatmaps keep their built-in scheme.
+- **\`defaultMode\` and \`allowUserToggle\` are instance-only.** No per-chart annotation controls the light/dark default or the toggle lock; set them in the config \`theme\` block or the editor.
+- **Environment-level theming is not applied yet.** Only the instance theme and the \`# theme.*\` / \`## theme.*\` per-chart annotations take effect today.` }, { name: "html-data-app-embedding", description: "Embed an in-package HTML data app into a host page or another application, including auto-sizing and auth. Read when embedding a Publisher page via Publisher.embed.", body: '# Embedding an HTML Data App\n\n> `Publisher.embed(selector, { src })` drops a package page into a host page as a sandboxed, auto-resizing iframe. Same-origin embeds authenticate with the browser\'s cookies; cross-origin embeds need a signed token.\n\n## The host-page pattern\n\n```html\n<script src="https://your-publisher/sdk/publisher.js"></script>\n<div id="dashboard"></div>\n<script>\n  const handle = Publisher.embed("#dashboard", {\n    src: "https://your-publisher/environments/demo/packages/sales/index.html",\n  });\n  // handle.destroy() removes the iframe and detaches its listeners.\n</script>\n```\n\n`embed(selector, options)` returns `{ iframe, destroy() }`. Options: `src` (required), `token` (a signed token for cross-origin auth, appended as `embed_token`), `height` (omit to auto-size; a number is treated as pixels), and `allow` (the iframe permissions policy).\n\n## Sizing and the resize contract\n\nOmit `height` and the frame auto-sizes. The embedded page measures its real content height and posts a `publisher:resize` message to the host, which resizes the iframe and accepts that message only from the iframe it created. You write none of this; it ships in `/sdk/publisher.js`, so the embedded page only has to load that script.\n\nDo not rely on `body { min-height: 100vh }` to drive the frame height. The runtime deliberately measures the content\'s bottom edge, not the viewport, to avoid a grow-forever loop.\n\n## Auth\n\n- Same-origin or same-tenant: pass no token. The browser\'s cookies authenticate the iframe.\n- Cross-origin: mint a short-lived signed token server-side and pass it as `options.token`. The runtime appends it to the iframe URL as `embed_token`; the embedded page must read it (from `location.search`) and call `Publisher.setToken(token)`. Because it rides in the URL, it can land in browser history, Referer headers, and server logs, so keep it short-lived and scoped to that one embed, and never put a long-lived or admin token in client HTML.\n\n## Guardrails (v1)\n\n- The iframe is sandboxed (`allow-scripts allow-same-origin allow-forms`). Design for that: no top-level navigation, no popups.\n- Embedded author JavaScript runs with the viewing user\'s data authority, so treat everything under `public/` as strictly first-party code: do not load untrusted third-party scripts, and do not move query results off to other hosts. Tighter per-embed isolation is planned.' }, { name: "html-data-app-runtime", description: "Write the JavaScript that drives an in-package HTML data app, calling Publisher.query, building queries from filter state, and handling results and errors. Read before writing the page's data code.", body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"carriers.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName`, `filterParams` (values for the model\'s legacy `#(filter)` source filters; `Publisher.query` does not forward Malloy `given:` values), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese run against the example `carriers` package (source `carriers`; views `by_letter`, `by_size_bucket`, `kpis`). Swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("carriers.malloy", "run: carriers -> by_letter");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n  const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n  const parts = [];\n  if (state.letter) parts.push(`letter = \'${q(state.letter)}\'`);\n  if (state.bucket) parts.push(`size_bucket = \'${q(state.bucket)}\'`);\n  return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n  "carriers.malloy",\n  `run: carriers -> by_letter + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input. A data-app page has no clean server-side parameterization for arbitrary input today: `Publisher.query` forwards `opts.filterParams` (the deprecated `#(filter)` source-filter API), not Malloy `given:` values. So constrain the input to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("carriers.malloy", "run: carriers -> kpis");\nel.textContent = kpis.total;   // the result is an array; kpis.total, not rows.total\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [byLetter, byBucket, kpisRows] = await Promise.all([\n  Publisher.query("carriers.malloy", "run: carriers -> by_letter"),\n  Publisher.query("carriers.malloy", "run: carriers -> by_size_bucket"),\n  Publisher.query("carriers.malloy", "run: carriers -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n  ```js\n  // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n  // date to a "YYYY-MM" string; monthLabel formats it for display.\n  // target may not cover every actual month; an absent month stays null, never 0.\n  const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n  const data = actualRows.map((r) => ({\n    label: monthLabel(r.order_month),\n    actual: Number(r.revenue),\n    target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n  }));\n  ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("carriers.malloy", "run: carriers -> by_letter")\n  .then((rows) => {\n    if (!rows.length) { el.textContent = "No data."; return; }\n    render(rows);\n  })\n  .catch((err) => {\n    el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n  });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("carriers.malloy", "run: carriers -> by_letter");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The shipped example renders rows with a plain chart library instead, so this path is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: letter` gives a `letter` column; `aggregate: n is count()` gives an `n` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"carriers.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400, required parameter | The model marks a runtime parameter (given) as required. Supply it via `opts.filterParams`, or pass `bypassFilters: true` only if you are a trusted caller. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |' }, { name: "html-data-apps", description: "Build or modify an in-package HTML data app for a Malloy Publisher package (a public/ directory the package serves). Use when the user wants a hand-authored HTML dashboard or web page backed by a package's Malloy models, with no build step.", body: `# In-Package HTML Data Apps
 
 > A package becomes a web app by adding a \`public/\` directory. Publisher serves those files and gives the page \`Publisher.query(...)\` to run Malloy against the package's models. No build step, no npm, no framework.
 
@@ -266436,7 +266623,7 @@ Use \`+\` to modify existing views: \`run: source -> my_view + { limit: 15, wher
 
 Step complete. Output: analysis \`.malloy\` file with views, insights, and reusable building blocks. For chart/renderer details, see \`skill:gotchas-rendering\` or call \`malloy_searchDocs\`. To formalize into a model, hand off to the modeling skill (\`skill:malloy-model\`).
 
-Publishing is out of scope for now: open-source Publisher serves the model from disk, and self-hosters publish via git plus their host's publish path.` }, { name: "malloy-charts", description: 'Chart selection guidance and renderer reference for Malloy views. Use when choosing visualization types, adding chart annotations, user asks "what chart should I use", "how should I visualize this", or when deciding between bar_chart, line_chart, scatter_chart, etc.', body: '# Chart Selection for Malloy\n\n> Malloy uses Vega-Lite under the hood. `#` tags control visualization. Call `malloy_searchDocs` with topic "rendering" for the full tag reference (or see https://docs.malloydata.dev/documentation/visualizations/overview).\n\n## Decision Tree: Which Chart?\n\n| Data Shape | Default Choice |\n|-----------|---------------|\n| Aggregates only (no group_by) | `# big_value` |\n| 1 time column + 1 measure | `# line_chart` |\n| 1 category + 1 measure | `# bar_chart` |\n| 2 numeric columns | `# scatter_chart` |\n| Geographic (US states) + 1 measure | `# shape_map` |\n| Route data (lat/lon pairs) | `# segment_map` |\n| Multiple perspectives | `# dashboard` with `nest:` |\n| Nested query to pivot | `# pivot` |\n| Filtered aggregates side-by-side | `# flatten` |\n| Detailed rows | Default table (no annotation) |\n\n| Goal | Renderer |\n|------|---------|\n| Compare categories | `# bar_chart` (sort by value, limit ~15) |\n| Show composition | `# bar_chart.stack` |\n| Trend over time | `# line_chart` |\n| Highlight KPIs | `# big_value` with `# label` |\n| Correlation | `# scatter_chart` |\n| Compare dimensions | `# dashboard` (nest chart views) |\n| Before/after | `# transpose` or `# pivot` |\n| Multiple metrics per category | Default table, `# flatten`, or `y=[\'a\',\'b\']` |\n\n**Constraints:**\n- ONE aggregate per chart view (charts render only the first; use `y=[\'a\',\'b\']` for multi-measure)\n- No fixed scale on measure definitions: use `# currency` not `# currency=usd0m`\n- One tag per line\n- Alias joined fields in `group_by` before `order_by`\n- Define measures in source, not in views\n\n\n## Chart Types\n\n### `# bar_chart`\n\n**Data shape:** `group_by` = x-axis, `aggregate` = y-axis, optional 2nd `group_by` = series.\n\n```malloy\n# bar_chart\nview: by_carrier is { group_by: carrier, aggregate: flight_count, order_by: flight_count desc, limit: 10 }\n\n# bar_chart.stack\nview: by_region is { group_by: category, region, aggregate: revenue }\n\n# bar_chart { y=[\'revenue\',\'cost\'] }\nview: rev_vs_cost is { group_by: category, aggregate: revenue, cost }\n```\n\n**Key properties:** `.stack`, `.size` (spark/xs/sm/md/lg/xl/2xl), `.x`, `.x.limit`, `.y` (supports `y=[\'a\',\'b\']`), `.series`, `.series.limit` (default 20), `.title`, `.subtitle`, `.x.independent`, `.y.independent`\n\n**Field role tags:** `# x`, `# y`, `# series` on individual fields to assign roles explicitly.\n\n### `# line_chart`\n\n**Data shape:** `group_by` (temporal/numeric) = x-axis, `aggregate` = y-axis, optional 2nd `group_by` = series.\n\n```malloy\n# line_chart\nview: trend is { group_by: order_month, aggregate: revenue, order_by: order_month }\n\n# line_chart { size=spark }\nview: mini_trend is { group_by: order_month, aggregate: revenue, order_by: order_month }\n```\n\n**Key properties:** `.zero_baseline`, `.interpolate` (e.g., `step`), `.size`, `.y` (supports `y=[\'a\',\'b\']`), `.series.limit` (default 12), `.title`, `.subtitle`\n\n### `# scatter_chart`\n\n**Data shape:** Fields by position: x, y, color, size (bubble), shape.\n\n```malloy\n# scatter_chart\nview: correlation is { group_by: customer_id, aggregate: avg_price, total_quantity }\n```\n\n### `# shape_map`\n\nChoropleth. US states only. Fields: state name, value.\n\n```malloy\n# shape_map\nview: by_state is { group_by: state, aggregate: revenue }\n```\n\n### `# segment_map`\n\nRoute map. US only. Fields: start_lat, start_lon, end_lat, end_lon, color.\n\n\n## Layout Types\n\n### `# big_value`\n\nKPI cards. Aggregates only, no `group_by`.\n\n```malloy\n# big_value\nview: summary is {\n  aggregate:\n    # label="Revenue"\n    # currency\n    revenue\n    # label="Orders"\n    # number=auto\n    order_count\n}\n```\n\n**Properties:** `.size`, `.sparkline=<nested_view_name>`, `.comparison_field`, `.comparison_label`, `.down_is_good`\n\n### `# dashboard`\n\nMulti-tile layout. Use `# break` to force new row.\n\n```malloy\n# dashboard\nview: overview is {\n  nest: # big_value\n    kpis is { ... }\n  nest: # line_chart\n    trend is { ... }\n  # break\n  nest: # bar_chart\n    breakdown is { ... }\n}\n```\n\n### `# pivot`\n\nPivot nested results into columns. Max 30 pivot columns.\n\n```malloy\nview: sales is {\n  group_by: product, aggregate: total\n  nest: # pivot\n    by_quarter is { group_by: quarter, aggregate: revenue }\n}\n```\n\n### `# transpose`\n\nSwap rows/columns. Good for period comparisons.\n\n```malloy\n# transpose\nview: comparison is {\n  aggregate:\n    # label="This Month"\n    current_revenue\n    # label="Last Month"\n    prior_revenue\n}\n```\n\n### `# list` / `# list_detail`\n\nList renders as comma-separated values. List_detail shows `value (detail)` pairs.\n\n### `# flatten`\n\nCollapse nested record into parent table as columns. Use for side-by-side filtered aggregates:\n\n```malloy\nview: segments is {\n  group_by: product, aggregate: total_revenue\n  nest: # flatten\n    enterprise is { where: segment = \'Enterprise\', aggregate: # label="Enterprise" revenue }\n  nest: # flatten\n    smb is { where: segment = \'SMB\', aggregate: # label="SMB" revenue }\n}\n```\n\n### `# table`\n\nDefault (implicit). Use explicitly for `.size=fill` property.\n\n\n## Field Formatting Tags\n\n| Tag | Use For | Shorthand |\n|-----|---------|-----------|\n| `# number` | Numeric formatting | `=auto` (K/M/B), `=id` (no commas), `=1k`, `=1m` |\n| `# percent` | Percentages | (none needed) |\n| `# currency` | Money | `=usd2m` (USD, 2 decimals, millions); scale only in views |\n| `# duration` | Time durations | `=seconds`, `=minutes`, `=hours`, `=days` |\n| `# data_volume` | Storage sizes | `=bytes`, `=kb`, `=mb`, `=gb` |\n| `# link` | Hyperlinks | `.url_template="https://example.com/$$"` |\n| `# image` | Inline images | `.height=40px`, `.width=100px` |\n\n**Currency codes:** `usd` ($), `eur`, `gbp`. **Scale:** K/M/B/T/Q or `auto`.\n**Number suffix styles:** `word` ("42.5 million"), `letter` ("42.5M"), `scientific`.\n\n## Utility Tags\n\n| Tag | Purpose |\n|-----|---------|\n| `# hidden` | Hide from output (still usable for sorting/references) |\n| `# label="..."` | Override display name |\n| `# description="..."` | Tooltip text |\n| `# tooltip` | Include nested view in chart tooltip |\n| `# break` | Force new dashboard row |\n| `# column { width=sm }` | Table column width |\n\n## Model-Level Defaults\n\n```malloy\n## viz.line_chart.defaults.y.independent=true\n## viz.bar_chart.defaults.stack\n## theme.fontFamily="Inter, sans-serif"\n```\n\n\n## Advanced Patterns\n\n### Sparklines in KPI Cards\n\n```malloy\n# big_value { sparkline=trend }\nview: revenue_kpi is {\n  aggregate: # label="Revenue" # currency revenue\n  nest: # line_chart { size=spark } # hidden\n    trend is { group_by: order_date, aggregate: revenue, order_by: order_date }\n}\n```\n\n### KPIs with Comparison Deltas\n\n```malloy\n# big_value { comparison_field=prior_month comparison_label="vs Last Month" }\nview: rev_delta is {\n  aggregate: # label="Revenue" # currency revenue, # hidden prior_month\n}\n```\n\nUse `down_is_good=true` for metrics where decrease is positive (churn, defects).\n\n### Inline Mini-Charts in Table Rows\n\n```malloy\nview: carriers is {\n  group_by: carrier, aggregate: flight_count\n  nest: # line_chart { size=spark }\n    trend is { group_by: month, aggregate: flight_count, order_by: month }\n}\n```\n\n### Multi-Measure Series\n\n```malloy\n# bar_chart { y=[\'revenue\',\'cost\'] }\nview: rev_vs_cost is { group_by: quarter, aggregate: revenue, cost }\n```\n\n### Hierarchical Drill-Down\n\n```malloy\n# list_detail\nview: explorer is {\n  group_by: region, aggregate: revenue\n  nest: # bar_chart\n    by_category is { group_by: category, aggregate: revenue, order_by: revenue desc, limit: 10 }\n}\n```\n\n### Distribution (Histogram)\n\nCall `malloy_searchDocs("autobin")` for syntax:\n```malloy\n# bar_chart\nview: price_dist is { group_by: bucket is autobin(price, 20), aggregate: order_count }\n```\n\n\n## Patterns for Missing Chart Types\n\n| Desired | Malloy Approximation |\n|---------|---------------------|\n| Pie/donut | `# bar_chart` sorted by value |\n| Treemap | Nested table with `order_by: desc` |\n| Heatmap | `# pivot` with color values |\n| Stacked area | `# line_chart` with series (overlaid lines) |\n| Funnel | `# bar_chart` with ordered stages |\n| Gauge/bullet | `# big_value` with `.comparison_field` |\n\n\n## Chart Annotations on Queries with `nest:`\n\nA top-level chart tag (e.g., `# bar_chart`) renders only the outer query; any `nest:` views are silently hidden from the rendering (still in raw data). To show nests, use `# dashboard` on the outer query with chart tags on each nest. Otherwise, drop the `nest:`.\n\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| Two aggregates in chart | ONE aggregate, or use `y=[\'a\',\'b\']` |\n| `# currency=usd0m` on measure | `# currency` (no scale) on defs; scale only in views |\n| Chart annotation on `nest:` line | Put on the **view definition** |\n| Tags on same line | One tag per line |\n| Sparkline not showing | Add `# hidden` to nested view AND reference in `.sparkline=` |\n| Pivot > 30 columns | Filter/limit the nested group_by |\n\nNOTE: The term \'constructor\' is a reserved term in Vega-Lite. If the word \'constructor\' appears in the query, it will cause the rendering to fail. Never use it in a query and avoid using it as a dimension in a model.\n\nFor more patterns, call `malloy_searchDocs` with topics like "bar charts", "line charts", "dashboards", "autobin", "percent of total", "comparing timeframes", or "pivots".\n\n## Further Reading\n\n- [Visualizations Overview](https://docs.malloydata.dev/documentation/visualizations/overview) - Official docs\n- [Bar Charts](https://docs.malloydata.dev/documentation/visualizations/bar_charts) - Stacked, grouped, series\n- [Bump Charts Blog](https://docs.malloydata.dev/blog/2023-10-26-malloy-bump-chart/) - Ranking over time\n- [Dataviz is Hierarchical](https://docs.malloydata.dev/blog/2024-02-29-hierarchical-viz/) - Nested data visualization philosophy' }, { name: "malloy-debug", description: 'Fix Malloy compile errors and understand error messages. Use when encountering errors in .malloy files, user says "fix this error", "malloy error", "compile error", "syntax error", or sees 20+ cascading errors.', body: `# Debugging Malloy Errors
+Publishing is out of scope for now: open-source Publisher serves the model from disk, and self-hosters publish via git plus their host's publish path.` }, { name: "malloy-charts", description: 'Chart selection guidance and renderer reference for Malloy views. Use when choosing visualization types, adding chart annotations, user asks "what chart should I use", "how should I visualize this", or when deciding between bar_chart, line_chart, scatter_chart, etc.', body: '# Chart Selection for Malloy\n\n> Malloy uses Vega-Lite under the hood. `#` tags control visualization. Call `malloy_searchDocs` with topic "rendering" for the full tag reference (or see https://docs.malloydata.dev/documentation/visualizations/overview).\n\n## Decision Tree: Which Chart?\n\n| Data Shape | Default Choice |\n|-----------|---------------|\n| Aggregates only (no group_by) | `# big_value` |\n| 1 time column + 1 measure | `# line_chart` |\n| 1 category + 1 measure | `# bar_chart` |\n| 2 numeric columns | `# scatter_chart` |\n| Geographic (US states) + 1 measure | `# shape_map` |\n| Route data (lat/lon pairs) | `# segment_map` |\n| Multiple perspectives | `# dashboard` with `nest:` |\n| Nested query to pivot | `# pivot` |\n| Filtered aggregates side-by-side | `# flatten` |\n| Detailed rows | Default table (no annotation) |\n\n| Goal | Renderer |\n|------|---------|\n| Compare categories | `# bar_chart` (sort by value, limit ~15) |\n| Show composition | `# bar_chart.stack` |\n| Trend over time | `# line_chart` |\n| Highlight KPIs | `# big_value` with `# label` |\n| Correlation | `# scatter_chart` |\n| Compare dimensions | `# dashboard` (nest chart views) |\n| Before/after | `# transpose` or `# pivot` |\n| Multiple metrics per category | Default table, `# flatten`, or `y=[\'a\',\'b\']` |\n\n**Constraints:**\n- ONE aggregate per chart view (charts render only the first; use `y=[\'a\',\'b\']` for multi-measure)\n- No fixed scale on measure definitions: use `# currency` not `# currency=usd0m`\n- One tag per line\n- Alias joined fields in `group_by` before `order_by`\n- Define measures in source, not in views\n\n\n## Chart Types\n\n### `# bar_chart`\n\n**Data shape:** `group_by` = x-axis, `aggregate` = y-axis, optional 2nd `group_by` = series.\n\n```malloy\n# bar_chart\nview: by_carrier is { group_by: carrier, aggregate: flight_count, order_by: flight_count desc, limit: 10 }\n\n# bar_chart.stack\nview: by_region is { group_by: category, region, aggregate: revenue }\n\n# bar_chart { y=[\'revenue\',\'cost\'] }\nview: rev_vs_cost is { group_by: category, aggregate: revenue, cost }\n```\n\n**Key properties:** `.stack`, `.size` (spark/xs/sm/md/lg/xl/2xl), `.x`, `.x.limit`, `.y` (supports `y=[\'a\',\'b\']`), `.series`, `.series.limit` (default 20), `.title`, `.subtitle`, `.x.independent`, `.y.independent`\n\n**Field role tags:** `# x`, `# y`, `# series` on individual fields to assign roles explicitly.\n\n### `# line_chart`\n\n**Data shape:** `group_by` (temporal/numeric) = x-axis, `aggregate` = y-axis, optional 2nd `group_by` = series.\n\n```malloy\n# line_chart\nview: trend is { group_by: order_month, aggregate: revenue, order_by: order_month }\n\n# line_chart { size=spark }\nview: mini_trend is { group_by: order_month, aggregate: revenue, order_by: order_month }\n```\n\n**Key properties:** `.zero_baseline`, `.interpolate` (e.g., `step`), `.size`, `.y` (supports `y=[\'a\',\'b\']`), `.series.limit` (default 12), `.title`, `.subtitle`\n\n### `# scatter_chart`\n\n**Data shape:** Fields by position: x, y, color, size (bubble), shape.\n\n```malloy\n# scatter_chart\nview: correlation is { group_by: customer_id, aggregate: avg_price, total_quantity }\n```\n\n### `# shape_map`\n\nChoropleth. US states only. Fields: state name, value.\n\n```malloy\n# shape_map\nview: by_state is { group_by: state, aggregate: revenue }\n```\n\n### `# segment_map`\n\nRoute map. US only. Fields: start_lat, start_lon, end_lat, end_lon, color.\n\n\n## Layout Types\n\n### `# big_value`\n\nKPI cards. Aggregates only, no `group_by`.\n\n```malloy\n# big_value\nview: summary is {\n  aggregate:\n    # label="Revenue"\n    # currency\n    revenue\n    # label="Orders"\n    # number=auto\n    order_count\n}\n```\n\n**Properties:** `.size`, `.sparkline=<nested_view_name>`, `.comparison_field`, `.comparison_label`, `.down_is_good`\n\n### `# dashboard`\n\nMulti-tile layout. Use `# break` to force new row.\n\n```malloy\n# dashboard\nview: overview is {\n  nest: # big_value\n    kpis is { ... }\n  nest: # line_chart\n    trend is { ... }\n  # break\n  nest: # bar_chart\n    breakdown is { ... }\n}\n```\n\n### `# pivot`\n\nPivot nested results into columns. Max 30 pivot columns.\n\n```malloy\nview: sales is {\n  group_by: product, aggregate: total\n  nest: # pivot\n    by_quarter is { group_by: quarter, aggregate: revenue }\n}\n```\n\n### `# transpose`\n\nSwap rows/columns. Good for period comparisons.\n\n```malloy\n# transpose\nview: comparison is {\n  aggregate:\n    # label="This Month"\n    current_revenue\n    # label="Last Month"\n    prior_revenue\n}\n```\n\n### `# list` / `# list_detail`\n\nList renders as comma-separated values. List_detail shows `value (detail)` pairs.\n\n### `# flatten`\n\nCollapse nested record into parent table as columns. Use for side-by-side filtered aggregates:\n\n```malloy\nview: segments is {\n  group_by: product, aggregate: total_revenue\n  nest: # flatten\n    enterprise is { where: segment = \'Enterprise\', aggregate: # label="Enterprise" revenue }\n  nest: # flatten\n    smb is { where: segment = \'SMB\', aggregate: # label="SMB" revenue }\n}\n```\n\n### `# table`\n\nDefault (implicit). Use explicitly for `.size=fill` property.\n\n\n## Field Formatting Tags\n\n| Tag | Use For | Shorthand |\n|-----|---------|-----------|\n| `# number` | Numeric formatting | `=auto` (K/M/B), `=id` (no commas), `=1k`, `=1m` |\n| `# percent` | Percentages | (none needed) |\n| `# currency` | Money | `=usd2m` (USD, 2 decimals, millions); scale only in views |\n| `# duration` | Time durations | `=seconds`, `=minutes`, `=hours`, `=days` |\n| `# data_volume` | Storage sizes | `=bytes`, `=kb`, `=mb`, `=gb` |\n| `# link` | Hyperlinks | `.url_template="https://example.com/$$"` |\n| `# image` | Inline images | `.height=40px`, `.width=100px` |\n\n**Currency codes:** `usd` ($), `eur`, `gbp`. **Scale:** K/M/B/T/Q or `auto`.\n**Number suffix styles:** `word` ("42.5 million"), `letter` ("42.5M"), `scientific`.\n\n## Utility Tags\n\n| Tag | Purpose |\n|-----|---------|\n| `# hidden` | Hide from output (still usable for sorting/references) |\n| `# label="..."` | Override display name |\n| `# description="..."` | Tooltip text |\n| `# tooltip` | Include nested view in chart tooltip |\n| `# break` | Force new dashboard row |\n| `# column { width=sm }` | Table column width |\n\n## Model-Level Defaults\n\n```malloy\n## viz.line_chart.defaults.y.independent=true\n## viz.bar_chart.defaults.stack\n```\n\n## Theming\n\nPublisher styles charts and tables from one structured theme. The instance sets it (in `publisher.config.json`\'s `theme` block or the **Settings, then Theme** editor); a model overrides it per result with `# theme.*` annotations, or model-wide with `## theme.*`. Per-chart annotations use the same nested `palette.*` / `font.*` vocabulary as the config, not flat key names, and they win over the instance theme for the keys they set. The forms:\n\n| Annotation | Controls | Modes |\n|-----------|----------|-------|\n| `# theme.palette.series` | Categorical series colors (array) | shared |\n| `# theme.palette.background.{light,dark}` | Chart canvas + table background | per-mode |\n| `# theme.palette.tableHeader.{light,dark}` | Table header text color | per-mode |\n| `# theme.palette.tableHeaderBackground.{light,dark}` | Table header row background | per-mode |\n| `# theme.palette.tableBody.{light,dark}` | Table body text color | per-mode |\n| `# theme.palette.tile.{light,dark}` | Dashboard tile background | per-mode |\n| `# theme.palette.tileTitle.{light,dark}` | Dashboard tile title color | per-mode |\n| `# theme.palette.mapColor.{light,dark}` | Choropleth gradient (`# shape_map` / `# segment_map`) | per-mode |\n| `# theme.font.family` | Font for all rendered text | shared |\n| `# theme.font.size` | Table font size (px) | shared |\n\nThe seven `palette.*` color keys each take a `.light` and/or `.dark` variant so dark mode gets its own value. `palette.series`, `font.family`, and `font.size` are single values shared across modes.\n\n```malloy\n// Model-wide defaults (## applies to every view in the model):\n## theme.palette.series = ["#14b3cb", "#e47404", "#1474a4"]\n## theme.font.family = "Inter, sans-serif"\n\n// Per-view override (# applies to this result only; beats the instance theme):\n# theme.palette.background.light = "#fafafa"\n# theme.palette.background.dark = "#111111"\n# theme.palette.tableHeader.dark = "#94a3b8"\nview: revenue_by_month is {\n  group_by: month\n  aggregate: revenue\n}\n```\n\n**Precedence**, highest to lowest, per key: `# theme.*` on the view, then `## theme.*` model default, then the instance theme, then Publisher\'s built-in defaults. A per-chart annotation overrides the instance for the keys it sets; unset keys fall through to the instance. (This is the reverse of a bare `@malloydata/render` embed, where the embedder wins: Publisher reads the annotation itself and layers it on top.)\n\nQuote values that contain spaces or a leading `#`. The light/dark default (`defaultMode`) and the toggle lock (`allowUserToggle`) are instance-only: set them in the config `theme` block or the editor, not as annotations. The gotchas-rendering skill lists the annotation forms that look valid but do nothing.\n\n\n## Advanced Patterns\n\n### Sparklines in KPI Cards\n\n```malloy\n# big_value { sparkline=trend }\nview: revenue_kpi is {\n  aggregate: # label="Revenue" # currency revenue\n  nest: # line_chart { size=spark } # hidden\n    trend is { group_by: order_date, aggregate: revenue, order_by: order_date }\n}\n```\n\n### KPIs with Comparison Deltas\n\n```malloy\n# big_value { comparison_field=prior_month comparison_label="vs Last Month" }\nview: rev_delta is {\n  aggregate: # label="Revenue" # currency revenue, # hidden prior_month\n}\n```\n\nUse `down_is_good=true` for metrics where decrease is positive (churn, defects).\n\n### Inline Mini-Charts in Table Rows\n\n```malloy\nview: carriers is {\n  group_by: carrier, aggregate: flight_count\n  nest: # line_chart { size=spark }\n    trend is { group_by: month, aggregate: flight_count, order_by: month }\n}\n```\n\n### Multi-Measure Series\n\n```malloy\n# bar_chart { y=[\'revenue\',\'cost\'] }\nview: rev_vs_cost is { group_by: quarter, aggregate: revenue, cost }\n```\n\n### Hierarchical Drill-Down\n\n```malloy\n# list_detail\nview: explorer is {\n  group_by: region, aggregate: revenue\n  nest: # bar_chart\n    by_category is { group_by: category, aggregate: revenue, order_by: revenue desc, limit: 10 }\n}\n```\n\n### Distribution (Histogram)\n\nCall `malloy_searchDocs("autobin")` for syntax:\n```malloy\n# bar_chart\nview: price_dist is { group_by: bucket is autobin(price, 20), aggregate: order_count }\n```\n\n\n## Patterns for Missing Chart Types\n\n| Desired | Malloy Approximation |\n|---------|---------------------|\n| Pie/donut | `# bar_chart` sorted by value |\n| Treemap | Nested table with `order_by: desc` |\n| Heatmap | `# pivot` with color values |\n| Stacked area | `# line_chart` with series (overlaid lines) |\n| Funnel | `# bar_chart` with ordered stages |\n| Gauge/bullet | `# big_value` with `.comparison_field` |\n\n\n## Chart Annotations on Queries with `nest:`\n\nA top-level chart tag (e.g., `# bar_chart`) renders only the outer query; any `nest:` views are silently hidden from the rendering (still in raw data). To show nests, use `# dashboard` on the outer query with chart tags on each nest. Otherwise, drop the `nest:`.\n\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| Two aggregates in chart | ONE aggregate, or use `y=[\'a\',\'b\']` |\n| `# currency=usd0m` on measure | `# currency` (no scale) on defs; scale only in views |\n| Chart annotation on `nest:` line | Put on the **view definition** |\n| Tags on same line | One tag per line |\n| Sparkline not showing | Add `# hidden` to nested view AND reference in `.sparkline=` |\n| Pivot > 30 columns | Filter/limit the nested group_by |\n\nNOTE: The term \'constructor\' is a reserved term in Vega-Lite. If the word \'constructor\' appears in the query, it will cause the rendering to fail. Never use it in a query and avoid using it as a dimension in a model.\n\nFor more patterns, call `malloy_searchDocs` with topics like "bar charts", "line charts", "dashboards", "autobin", "percent of total", "comparing timeframes", or "pivots".\n\n## Further Reading\n\n- [Visualizations Overview](https://docs.malloydata.dev/documentation/visualizations/overview) - Official docs\n- [Bar Charts](https://docs.malloydata.dev/documentation/visualizations/bar_charts) - Stacked, grouped, series\n- [Bump Charts Blog](https://docs.malloydata.dev/blog/2023-10-26-malloy-bump-chart/) - Ranking over time\n- [Dataviz is Hierarchical](https://docs.malloydata.dev/blog/2024-02-29-hierarchical-viz/) - Nested data visualization philosophy' }, { name: "malloy-debug", description: 'Fix Malloy compile errors and understand error messages. Use when encountering errors in .malloy files, user says "fix this error", "malloy error", "compile error", "syntax error", or sees 20+ cascading errors.', body: `# Debugging Malloy Errors
 
 ## Get Diagnostics
 
@@ -266596,7 +266783,7 @@ extend {
   dimension:
     // Use dimension (not rename:) for cleaner column names
     order_type is \`Type\`
-    full_name is first_name || ' ' || last_name
+    full_name is concat(first_name, ' ', last_name)
     segment is lifetime_value ?
       pick 'enterprise' when >= 100000
       pick 'mid-market' when >= 10000
@@ -266607,6 +266794,8 @@ extend {
 }
 \`\`\`
 
+> **Every \`dimension:\` needs \`name is expr\`.** A bare column name like \`dimension: species\` is a parse error (e.g. \`missing IS at ...\`). Raw columns are already usable in \`group_by\` / \`select\` without any declaration, so only add a \`dimension:\` when deriving or renaming a field (e.g. \`revenue is price * quantity\`).
+
 ### Base Source (Curated Mode with Access Modifiers)
 
 \`\`\`malloy
@@ -266711,6 +266900,7 @@ source: customer_health is customers extend {
 
 ## Key Rules
 
+- **Every \`dimension:\` needs \`name is expr\`**: a bare \`dimension: species\` is a parse error. Raw columns are queryable directly in \`group_by\` / \`select\`; only declare a \`dimension:\` to derive or rename a field.
 - **Define joined tables before referencing them**, use \`import\` statements in multi-file architecture
 - **Use \`nullif(denominator, 0)\` for all division**
 - **Alias joined fields before using in \`order_by\`**: \`group_by: yr is table.year\`
@@ -268174,7 +268364,9 @@ class MaterializationService {
     this.runInBackground(created.id, (signal) => this.runBuild(created.id, environmentName, packageName, {
       sourceNames: options.sourceNames,
       forceRefresh,
-      buildInstructions
+      buildInstructions,
+      referenceManifest: options.referenceManifest,
+      strictUpstreams: options.strictUpstreams
     }, signal));
     return created;
   }
@@ -268199,12 +268391,12 @@ class MaterializationService {
       let carried;
       if (orchestrated) {
         instructions = opts.buildInstructions;
-        carried = {};
+        carried = this.referenceManifestToEntries(opts.referenceManifest);
       } else {
         const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
         ({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
       }
-      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
+      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal, opts.strictUpstreams ?? false);
       const sourcesBuilt = instructions.length;
       const sourcesReused = Object.keys(carried).length;
       const durationMs = Date.now() - startedAt;
@@ -268263,6 +268455,16 @@ class MaterializationService {
     }
     return { instructions, carried };
   }
+  referenceManifestToEntries(referenceManifest) {
+    const entries = {};
+    for (const ref of referenceManifest ?? []) {
+      entries[ref.sourceEntityId] = {
+        sourceEntityId: ref.sourceEntityId,
+        physicalTableName: ref.physicalTableName
+      };
+    }
+    return entries;
+  }
   async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
     const list = await this.repository.listMaterializations(environmentId, packageName);
     for (const m of list) {
@@ -268315,7 +268517,7 @@ class MaterializationService {
       }
     }
   }
-  async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
+  async executeInstructedBuild(compiled, instructions, seedEntries, signal, strict = false) {
     const { graphs, sources, connectionDigests, connections } = compiled;
     const bySourceID = new Map;
     const bySourceEntityId = new Map;
@@ -268326,6 +268528,7 @@ class MaterializationService {
       bySourceEntityId.set(instruction.sourceEntityId, instruction);
     }
     const manifest = new Manifest;
+    manifest.strict = strict;
     const entries = {};
     for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
       if (entry.physicalTableName) {