@malloy-publisher/server 0.2.1 → 0.2.2

package/dist/server.mjs

@@ -213188,7 +213188,7 @@ var require_utils74 = __commonJS((exports, module) => {
 
 // ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
 var require_sshcrypto = __commonJS((exports, module) => {
-  module.exports = __require("./sshcrypto-vd2k5hq9.node");
+  module.exports = __require("./sshcrypto-xqan60jb.node");
 });
 
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
@@ -248769,6 +248769,15 @@ function recordAutoLoadOutcome(outcome) {
 function recordConnectionDigestSkipped() {
   connectionDigestSkipCounter().add(1);
 }
+function recordDuplicateTargetSkipped() {
+  duplicateTargetSkipCounter().add(1);
+}
+function recordSharedAddressInstructions() {
+  sharedAddressInstructionCounter().add(1);
+}
+function recordTableCollision() {
+  tableCollisionCounter().add(1);
+}
 function recordManifestBind(outcome) {
   manifestBindCounter().add(1, { outcome });
 }
@@ -248808,7 +248817,7 @@ function recordChainedStorageBuild(outcome) {
 function recordColocatedBindDropped(reason) {
   colocatedBindDroppedCounter().add(1, { reason });
 }
-var resetHooks2, runCounter, runDuration, sourcesCounter, incrementalStepCounter, buildPlanComputeDuration, buildPlanComputeFailedCounter, autoLoadCounter, connectionDigestSkipCounter, manifestBindCounter, manifestBindDegradedCounter, sourceBuildDuration, dropTablesCounter, scheduledFireCounter, storageServeRoutingCounter, storageTableRetainedCounter, storageBuildFailureCounter, attributionSkippedCounter, eligibilityRefusedCounter, serveShapeTierDropCounter, serveShapeTypeFallbackCounter, chainedStorageBuildCounter, colocatedBindDroppedCounter;
+var resetHooks2, runCounter, runDuration, sourcesCounter, incrementalStepCounter, buildPlanComputeDuration, buildPlanComputeFailedCounter, autoLoadCounter, connectionDigestSkipCounter, manifestBindCounter, manifestBindDegradedCounter, duplicateTargetSkipCounter, sharedAddressInstructionCounter, tableCollisionCounter, sourceBuildDuration, dropTablesCounter, scheduledFireCounter, storageServeRoutingCounter, storageTableRetainedCounter, storageBuildFailureCounter, attributionSkippedCounter, eligibilityRefusedCounter, serveShapeTierDropCounter, serveShapeTypeFallbackCounter, chainedStorageBuildCounter, colocatedBindDroppedCounter;
 var init_materialization_metrics = __esm(() => {
   init_telemetry();
   resetHooks2 = [];
@@ -248822,10 +248831,13 @@ var init_materialization_metrics = __esm(() => {
   connectionDigestSkipCounter = lazyCounter2("publisher_materialization_connection_digest_skipped_total", "Connection digests skipped during build-plan compile because the connection did not resolve.");
   manifestBindCounter = lazyCounter2("publisher_materialization_manifest_bind_total", "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').");
   manifestBindDegradedCounter = lazyCounter2("publisher_materialization_manifest_bind_degraded_total", "Manifest entries bound with an UNQUOTED table path because their connection " + "could not be resolved (serve-side bind) or is absent from the build " + "(build-side seed). A misconfiguration that breaks the source on a " + "case-folding engine (Snowflake); alertable.");
+  duplicateTargetSkipCounter = lazyCounter2("publisher_materialization_duplicate_target_skipped_total", "Sources skipped because the physical table they name was already built in " + "this run. Ordinary for a package that extends a persisted source; a " + "rising count against a package with no extension means the plan is " + "enumerating one table under more names than expected.");
+  sharedAddressInstructionCounter = lazyCounter2("publisher_materialization_shared_address_instructions_total", "Content addresses that arrived with more than one instruction naming a " + "DIFFERENT physical table. The host minted a table per source where " + "several sources share one artifact. With a sourceID on each instruction " + "every table is built and only one is recorded, so the rest are orphaned; " + "without one the last instruction wins and the earlier names are never " + "built. Wasteful, not wrong — the table's CONTENT is the same either way.");
+  tableCollisionCounter = lazyCounter2("publisher_materialization_table_collision_total", "Two definitions with DIFFERENT content addresses materializing into ONE " + "physical table. Each build overwrites the other's rows while both " + "addresses resolve to the table at serve time, so a query is answered " + "from another source's data. A wrong answer, not wasted work — page on " + "this one. Refused instead of counted-and-continued when " + "PERSIST_COLLISION_ENFORCE is set, so a non-zero rate here is also the " + "measure of what flipping that flag would start refusing.");
   sourceBuildDuration = lazyHistogram("publisher_materialization_source_build_duration_ms", "Wall-clock duration of building a single persist source.", "ms");
   dropTablesCounter = lazyCounter2("publisher_materialization_drop_tables_total", "Physical tables dropped on delete. Label: outcome ('success'|'failure').");
   scheduledFireCounter = lazyCounter2("publisher_materialization_scheduled_fires_total", "Standalone-scheduler attempts to fire a package's materialization.schedule. " + "Label: outcome ('fired'|'conflict'|'error').");
-  storageServeRoutingCounter = lazyCounter2("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|" + "'runtime_live_fallback'|'blocked_by_row_level_gate').");
+  storageServeRoutingCounter = lazyCounter2("publisher_storage_serve_routing_total", "storage= serve routing decisions. Label: outcome ('storage'|'live_fallback'|" + "'runtime_live_fallback'|'blocked_by_row_level_gate'). Covers the storage= " + "tier only; a colocated #@ persist hit is in neither the numerator nor the " + "denominator. NOTE 'live_fallback' here means the transform was INELIGIBLE, " + "which QueryResult.servedFrom reports as null - that field's " + "'live_fallback' is this counter's 'runtime_live_fallback'.");
   storageTableRetainedCounter = lazyCounter2("publisher_storage_tables_retained_total", "Tables a FAILED run left in a storage= destination and deliberately did not " + "reclaim, because the source is refreshed incrementally and the name may be " + "the one it serves from. Label: destination. Not all of these are orphans — " + "a rebuild at a fresh generational name is, a seed on the live serving name " + "is not, and the manifest entry cannot separate them — so read this as an " + "upper bound on what is accumulating rather than a leak count. It is the " + "only accounting there is until reclaiming a destination exists, which is " + "why it is a counter and not just a log line: the question is a rate, not " + "whether it ever happened.");
   storageBuildFailureCounter = lazyCounter2("publisher_storage_build_failures_total", "storage= build failures (federation/passthrough/attach/CTAS), distinct from " + "in-warehouse build failures. Labels: destination (connection name), " + "reason ('build_failed'|'billed_read_not_captured'). The second is the " + "expensive one: the warehouse read ran and was charged, and the rows could " + "not be captured — so a re-drive pays for it again. Worth alerting on " + "separately from a failure that costs only a retry.");
   attributionSkippedCounter = lazyCounter2("publisher_storage_build_attribution_skipped_total", "storage= builds whose warehouse read went out UNATTRIBUTED while tagging was " + "on. Label: reason ('job_listing_unavailable'|'tag_failed'|" + "'read_row_not_found'|'read_row_ambiguous'|'cost_query_failed'). " + "The read still ran and the " + "build still succeeded — what was lost is the label in the customer's own " + "query history, and the cost on this side. Without this an operator who " + "turns tagging on and sees nothing has a single log line to go on.");
@@ -248914,8 +248926,34 @@ import {
   InMemoryURLReader,
   Runtime
 } from "@malloydata/malloy";
-function deriveServeBindings(entries) {
+function groupAliasesByName(planSources) {
+  const namesByAddress = new Map;
+  for (const source of planSources) {
+    if (!source.sourceEntityId || !source.name)
+      continue;
+    const group = namesByAddress.get(source.sourceEntityId);
+    if (!group)
+      namesByAddress.set(source.sourceEntityId, [source.name]);
+    else if (!group.includes(source.name))
+      group.push(source.name);
+  }
+  const addressesPerName = new Map;
+  for (const group of namesByAddress.values()) {
+    for (const name of group) {
+      addressesPerName.set(name, (addressesPerName.get(name) ?? 0) + 1);
+    }
+  }
+  const byName = {};
+  for (const group of namesByAddress.values()) {
+    const unambiguous = group.filter((name) => addressesPerName.get(name) === 1);
+    for (const name of unambiguous)
+      byName[name] = unambiguous;
+  }
+  return byName;
+}
+function deriveServeBindings(entries, aliasesBySourceName) {
   const bindings = [];
+  const builders = new Set(Object.values(entries).map((entry) => entry.sourceName).filter((name) => !!name));
   for (const entry of Object.values(entries)) {
     if (!entry.sourceName || !entry.storageDestinationName || !entry.physicalTableName) {
       continue;
@@ -248923,16 +248961,22 @@ function deriveServeBindings(entries) {
     const schema = (entry.schema ?? []).filter((c) => c.name && c.type).map((c) => ({ name: c.name, type: c.type }));
     if (schema.length === 0)
       continue;
-    bindings.push({
-      sourceName: entry.sourceName,
-      destinationName: entry.storageDestinationName,
-      virtualHandle: entry.sourceEntityId,
-      tablePath: `${entry.storageDestinationName}.${entry.physicalTableName}`,
-      schema,
-      freshAsOf: entry.dataAsOf,
-      freshnessWindowSeconds: entry.freshnessWindowSeconds,
-      freshnessFallback: entry.freshnessFallback
-    });
+    const names = [
+      entry.sourceName,
+      ...(aliasesBySourceName[entry.sourceName] ?? []).filter((name) => !builders.has(name))
+    ].filter((name, i, all3) => all3.indexOf(name) === i);
+    for (const sourceName of names) {
+      bindings.push({
+        sourceName,
+        destinationName: entry.storageDestinationName,
+        virtualHandle: entry.sourceEntityId,
+        tablePath: `${entry.storageDestinationName}.${entry.physicalTableName}`,
+        schema,
+        freshAsOf: entry.dataAsOf,
+        freshnessWindowSeconds: entry.freshnessWindowSeconds,
+        freshnessFallback: entry.freshnessFallback
+      });
+    }
   }
   return bindings;
 }
@@ -249040,6 +249084,14 @@ ${fields}
 }
 ${source}`;
 }
+function serveShapeDiagnostics(allBindings, freshBindings) {
+  const shapeSources = freshBindings.map((b) => b.sourceName);
+  const fresh = new Set(shapeSources);
+  return {
+    shapeSources,
+    staleSources: allBindings.map((b) => b.sourceName).filter((name) => !fresh.has(name))
+  };
+}
 function buildServeShapeModelForBindings(bindings) {
   const fragments = orderBindingsByJoinDeps(bindings).map(serveShapeFragment).join(`
 `);
@@ -251816,6 +251868,7 @@ var init_model = __esm(() => {
   init_gate_dimension();
   init_filter();
   init_given();
+  init_dashboard();
   init_motly();
   init_model_limits();
   init_json_utils();
@@ -252812,14 +252865,14 @@ var init_model = __esm(() => {
         } catch {
           continue;
         }
-        const errors2 = validateRenderTags2(result).filter((log) => log.severity === "error");
-        if (errors2.length > 0) {
-          logger.warn(`Invalid renderer configuration on '${target.label}': ${errors2.map((e) => e.message).join("; ")}`);
-          for (const e of errors2) {
+        const logs = filterPublisherOwnedRenderLogs(validateRenderTags2(result), this.modelPath);
+        if (logs.length > 0) {
+          logger.warn(`Render tag findings on '${target.label}': ${logs.map((e) => `[${e.severity}] ${e.message}`).join("; ")}`);
+          for (const e of logs) {
             findings.push({
               subject: target.label,
               message: e.message,
-              severity: "error"
+              severity: e.severity === "error" ? "error" : "warn"
             });
           }
         }
@@ -253088,9 +253141,12 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
             });
           } catch (shapeErr) {
             recordStorageServeRouting("live_fallback");
+            const { shapeSources, staleSources } = serveShapeDiagnostics(this.serveBindings, this.freshServeBindings(Date.now()));
             logger.info("storage serve-shape ineligible for this query; serving live", {
               modelPath: this.modelPath,
-              error: shapeErr instanceof Error ? shapeErr.message : String(shapeErr)
+              error: shapeErr instanceof Error ? shapeErr.message : String(shapeErr),
+              shapeSources,
+              staleSources
             });
           }
         }
@@ -286818,7 +286874,7 @@ class Package {
     this.recordManifestBinding(allowed);
   }
   bindStorageServeBindings(entries) {
-    const derived = deriveServeBindings(entries);
+    const derived = deriveServeBindings(entries, groupAliasesByName(Object.values(this.buildPlan?.sources ?? {})));
     const eligibility = this.sourceEligibility;
     const eligible = new Set(eligibility?.eligible ?? []);
     const allowed = derived.filter((binding) => {
@@ -293454,10 +293510,12 @@ var SEARCH_DOCS_DESCRIPTION = `Search the Malloy documentation by keyword and re
 ## When to use
 - Before writing unfamiliar Malloy syntax (window functions, autobin, dialect-specific functions, rendering tags) or when a query fails with a syntax error you do not recognize.
 - Do NOT use it to look up field or source names in a model; use malloy_getContext for that.
+- Do NOT use it for anything about running Publisher itself — server flags, deployment, connection or embedding-provider configuration, publisher.json, packages, watch mode. This index covers the Malloy LANGUAGE docs only. Those answers live in the deployment's own docs/ directory and bundled skills, not here.
 
 ## Contract rules
 - These are documentation pages, not model entities. Do not treat a doc title as a field or source name.
 - The excerpt is only a hint; open the url for the full detail.
+- Matching is keyword-based, so a result is not evidence the topic is covered. An off-topic query still returns its best keyword matches, and a title can match on a word it shares with your question while the page is about something else. Read the excerpt before trusting a hit, and treat a page-full of near-misses as "not documented here" rather than retrying with more keywords.
 
 ## Parameters
 - query (required): keywords describing what you need.
@@ -293822,7 +293880,7 @@ async function getPackageIndex(environmentStore, environmentName, packageName) {
   });
   return built;
 }
-var GET_CONTEXT_DESCRIPTION = `Discover what a Publisher deployment exposes and retrieve the model entities most relevant to a plain-English question, so you can ground a query in what the model actually defines instead of guessing. This is the starting point when you do not yet know the environment, package, or model names.
+var GET_CONTEXT_DESCRIPTION = `Discover what a Publisher deployment exposes and retrieve the model entities most relevant to a plain-English question, so you ground a query in what the model defines, not a guess. Start here when you do not know those names.
 
 ## Contract rules
 - Use the names it returns verbatim; never invent an environment, package, or entity that is not in the results.
@@ -293830,16 +293888,16 @@ var GET_CONTEXT_DESCRIPTION = `Discover what a Publisher deployment exposes and
 - An error, stale, or note field means the data did not load or predates the files: read it before trusting a number.
 
 ## Parameters
-All optional. Supply what you know and omit the rest; each combination answers at its own level.
+All optional; supply what you know. Each combination answers at its own level.
 - none: lists the environments, each with its package names.
 - environmentName: lists that environment's packages, with descriptions.
 - + packageName: lists that package's sources.
-- + query: a plain-English description of what you need, returning the sources, views, named queries, and dimension/measure fields most relevant to it.
-- sourceName: narrows retrieval to one source (the drill-down phase).
-- limit: caps results (max 50). Retrieval defaults to 10; the listing levels return all unless set.
+- + query: a plain-English description of what you need; returns the most relevant sources, views, queries, and dimension/measure fields.
+- sourceName: narrows to one source. Without a query it lists that source and its fields, views and queries, led by the source's own row, so [] means no such source. With a query it ranks within that source, so [] means nothing matched, not a missing source.
+- limit: caps results (max 50). Retrieval defaults to 10; listing levels return all unless set. The drill-down's source row counts.
 
 ## Response
-A JSON object with a results array whose items carry a kind field. For retrieval, each entity has kind (source / view / query / dimension / measure), name, source, modelPath, and doc; environmentName, packageName, modelPath, and source map directly onto malloy_executeQuery parameters, and for a view or named query you pass its name as queryName with sourceName. When the server is configured with an embedding provider, retrieval is ranked by semantic similarity: the payload then carries a retrieval field ("semantic", or "lexical" when the provider is unavailable) and each semantic entity a score.
+A JSON object with a results array. Each entity has kind (source / view / query / dimension / measure), name, source, modelPath, and doc; environmentName, packageName, modelPath, and source map onto malloy_executeQuery; pass a view or query as queryName with sourceName. With an embedding provider, retrieval is ranked semantically: the payload carries a retrieval field ("semantic", or "lexical" if it is down) plus a per-entity score. With no provider both are absent, not an error.
 
 ## Worked example
 { "environmentName": "examples", "packageName": "storefront", "query": "revenue by product category" }`;
@@ -293943,7 +294001,7 @@ function registerGetContextTool(mcpServer, environmentStore) {
     };
     const sanitized = query ? sanitize2(query) : "";
     if (!sanitized) {
-      const results2 = Array.from(byId.values()).filter((e) => e.kind === "source").filter((e) => !sourceName || e.source === sourceName).slice(0, limit).map((e) => ({
+      const results2 = Array.from(byId.values()).filter((e) => sourceName ? e.source === sourceName : e.kind === "source").slice(0, limit).map((e) => ({
         kind: e.kind,
         name: e.name,
         source: e.source,
@@ -294888,7 +294946,7 @@ End with a short **Next steps**: one or two specific deeper analyses the data co
     {
       name: "malloy-analysis-report",
       description: 'Combine validated Malloy queries into a notebook report or dashboard. Use when the user asks to "create a report", "build a dashboard", "combine these into a report", or wants a persistent multi-query artifact.',
-      body: "# Creating Reports\n\nAn ad-hoc report is a `.malloynb` notebook that combines markdown narrative with live Malloy query cells. There is no dedicated report tool: you author the notebook directly. Load `skill:malloy-notebooks` for the full `.malloynb` cell format and authoring rules; this skill covers when to build one and how to design good report content (cells, chart annotations, narrative structure).\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Before building a report\n\n1. **Run each query first** via `execute_query` to verify it works and returns expected results.\n2. **Explain the results** to the user as you go: walk through the analysis step by step.\n3. **Then assemble the notebook** once the analysis is validated.\n\nDo NOT build the notebook in the same turn as `execute_query`. Explain first, then build.\n\n## Filters are inherited from the model, don't declare them in the report\n\nReports do not (and cannot) define their own filters. If the source declares `given:` parameters (or legacy `#(filter)` annotations), Publisher renders the controls, parses caller parameters, and applies them server-side automatically: the report inherits and displays them with no extra work. If the analysis needs a knob the source doesn't expose, the right move is to add a `given:` to the source itself, not to wedge a filter widget into the report. `#(filter)` is deprecated in favour of native Malloy `given:` parameters. Do not add new `#(filter)` annotations; the two exceptions are `required` and `implicit`, which `given:` cannot cover yet. See `skill:malloy-model` § Legacy: Parameterizable Filters. For curated notebooks with their own per-notebook filter UI on top of the model, see `skill:malloy-notebooks` instead.\n\n## What goes in the report\n\nDo NOT add an H1 heading in any cell (use H2 and below for sections); the notebook name serves as the title. To redo the structure rather than tweak one cell, rewrite the notebook file end-to-end.\n\nMarkdown cells own narrative; query cells own a single Malloy query whose chart annotation tells the renderer how to display the result. Markdown supports H2 headings, lists, bold, and inline code. Keep narrative cells short, one idea per cell, so the rendered output reads as a story instead of a wall of text.\n\nIn a `.malloynb` file each cell is delimited by a `>>>markdown` or `>>>malloy` marker. A markdown cell looks like:\n\n```\n>>>markdown\n## Section heading\nNarrative text here.\n```\n\nA query cell looks like:\n\n```\n>>>malloy\n# bar_chart\nrun: source -> { group_by: dim; aggregate: measure }\n```\n\nEach Malloy cell must be a standalone query (for example `run: source -> { ... }`). The notebook's leading `>>>malloy` cell holds the `import` statement for the model file; individual query cells do not repeat it. If a query fails validation when executed, fix it and rerun.\n\nA well-structured report typically follows this pattern:\n\n```\n[Markdown]  ## Overview: what question are we answering, what data is in scope (date range, entity count)\n[Malloy]    KPI cell: headline numbers (e.g., # big_value, or # dashboard with nested # big_value cells)\n[Markdown]  ## Trend: describe what we should look for over time\n[Malloy]    Time-series cell (e.g., # line_chart on a date dimension)\n[Markdown]  ## Breakdown: where the signal is\n[Malloy]    Categorical cell (e.g., # bar_chart on a categorical dimension)\n[Markdown]  ## Key takeaways: what the user should walk away with\n```\n\nUse this as a default; deviate when the analysis warrants. A grounded report names the time range and entity count up front so every number that follows has context.\n\n## Choosing chart types and annotations\n\nRead `skill:malloy-charts` before picking visualizations: it owns chart-type selection, properties, and the placement rules for chart annotations. `skill:malloy-queries` covers Malloy query patterns and the critical placement rules for chart-annotation tags.\n\nWhen in doubt:\n- KPIs / single numbers -> `# big_value`, often nested inside `# dashboard`.\n- Trend over time -> `# line_chart`, usually on the primary date dimension.\n- Category comparisons -> `# bar_chart`, ordered by the metric.\n- Tabular data with many columns -> a plain table cell with `# table.size=fill`.\n- Multiple coordinated charts -> `# dashboard` with `nest:` blocks.\n\nAnnotations go **before** `run:`, never inside curly braces:\n\n```malloy\n# bar_chart\nrun: source -> {\n  group_by: category\n  aggregate: revenue\n  order_by: revenue desc\n  limit: 10\n}\n```\n\nA `# dashboard` cell composes nested views, useful for KPIs alongside a trend in a single cell. Each `nest:` is a tile; any top-level `aggregate:` measures render as KPI cards. For a fixed grid, use `# dashboard { columns=N }` with `# colspan` on each tile (see `skill:malloy-charts`):\n\n```malloy\n# dashboard { columns=2 }\nrun: source -> {\n  nest:\n    # colspan=2\n    # big_value\n    kpis is {\n      aggregate:\n        # label=\"Revenue\"\n        # currency\n        total_revenue\n\n        # label=\"Orders\"\n        # number=auto\n        order_count\n    }\n  nest:\n    # line_chart\n    trend is {\n      group_by: order_date.month\n      aggregate: total_revenue\n      order_by: 1\n    }\n}\n```\n\nKey rendering rules to keep in mind when shaping a cell:\n- FIRST `group_by` = x-axis, FIRST `aggregate` = y-axis.\n- Override field roles with `# x`, `# y`, `# series` on individual fields.\n- For multiple measure series, place `# y` above the `aggregate:` keyword.\n- One aggregate per chart view: use `# dashboard` with nested views for multiple charts.\n- Use `# table.size=fill` for standalone table queries.\n\n## Editing an existing report\n\nFor small targeted changes (fix one cell, insert one new cell), edit that cell in the `.malloynb` file rather than recreating the whole notebook. For structural rewrites (reordering many cells, changing the narrative arc), rewrite the notebook file.\n\n## IMPORTANT\n\nYou CANNOT see the rendered output of notebook cells. Do not claim to see charts, values, or patterns from report cells you haven't explicitly executed via `execute_query`. If you need to analyze results, run the query via `execute_query` first."
+      body: "# Creating Reports\n\nAn ad-hoc report is a `.malloynb` notebook that combines markdown narrative with live Malloy query cells. There is no dedicated report tool: you author the notebook directly. Load `skill:malloy-notebooks` for the full `.malloynb` cell format and authoring rules; this skill covers when to build one and how to design good report content (cells, chart annotations, narrative structure).\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Before building a report\n\n1. **Run each query first** via `execute_query` to verify it works and returns expected results.\n2. **Explain the results** to the user as you go: walk through the analysis step by step.\n3. **Then assemble the notebook** once the analysis is validated.\n\nDo NOT build the notebook in the same turn as `execute_query`. Explain first, then build.\n\n## Filters are inherited from the model, don't declare them in the report\n\nReports do not (and cannot) define their own filters. If the source declares `given:` parameters (or legacy `#(filter)` annotations), Publisher renders the controls, parses caller parameters, and applies them server-side automatically: the report inherits and displays them with no extra work. If the analysis needs a knob the source doesn't expose, the right move is to add a `given:` to the source itself, not to wedge a filter widget into the report. `#(filter)` is deprecated in favour of native Malloy `given:` parameters. Do not add new `#(filter)` annotations; the two exceptions are `required` and `implicit`, which `given:` cannot cover yet. The `malloy-model` skill covers this under § Legacy: Parameterizable Filters. For curated notebooks with their own per-notebook filter UI on top of the model, see `skill:malloy-notebooks` instead.\n\n## What goes in the report\n\nDo NOT add an H1 heading in any cell (use H2 and below for sections); the notebook name serves as the title. To redo the structure rather than tweak one cell, rewrite the notebook file end-to-end.\n\nMarkdown cells own narrative; query cells own a single Malloy query whose chart annotation tells the renderer how to display the result. Markdown supports H2 headings, lists, bold, and inline code. Keep narrative cells short, one idea per cell, so the rendered output reads as a story instead of a wall of text.\n\nIn a `.malloynb` file each cell is delimited by a `>>>markdown` or `>>>malloy` marker. A markdown cell looks like:\n\n```\n>>>markdown\n## Section heading\nNarrative text here.\n```\n\nA query cell looks like:\n\n```\n>>>malloy\n# bar_chart\nrun: source -> { group_by: dim; aggregate: measure }\n```\n\nEach Malloy cell must be a standalone query (for example `run: source -> { ... }`). The notebook's leading `>>>malloy` cell holds the `import` statement for the model file; individual query cells do not repeat it. If a query fails validation when executed, fix it and rerun.\n\nA well-structured report typically follows this pattern:\n\n```\n[Markdown]  ## Overview: what question are we answering, what data is in scope (date range, entity count)\n[Malloy]    KPI cell: headline numbers (e.g., # big_value, or # dashboard with nested # big_value cells)\n[Markdown]  ## Trend: describe what we should look for over time\n[Malloy]    Time-series cell (e.g., # line_chart on a date dimension)\n[Markdown]  ## Breakdown: where the signal is\n[Malloy]    Categorical cell (e.g., # bar_chart on a categorical dimension)\n[Markdown]  ## Key takeaways: what the user should walk away with\n```\n\nUse this as a default; deviate when the analysis warrants. A grounded report names the time range and entity count up front so every number that follows has context.\n\n## Choosing chart types and annotations\n\nRead `skill:malloy-charts` before picking visualizations: it owns chart-type selection, properties, and the placement rules for chart annotations. `skill:malloy-queries` covers Malloy query patterns and the critical placement rules for chart-annotation tags.\n\nWhen in doubt:\n- KPIs / single numbers -> `# big_value`, often nested inside `# dashboard`.\n- Trend over time -> `# line_chart`, usually on the primary date dimension.\n- Category comparisons -> `# bar_chart`, ordered by the metric.\n- Tabular data with many columns -> a plain table cell with `# table.size=fill`.\n- Multiple coordinated charts -> `# dashboard` with `nest:` blocks.\n\nAnnotations go **before** `run:`, never inside curly braces:\n\n```malloy\n# bar_chart\nrun: source -> {\n  group_by: category\n  aggregate: revenue\n  order_by: revenue desc\n  limit: 10\n}\n```\n\nA `# dashboard` cell composes nested views, useful for KPIs alongside a trend in a single cell. Each `nest:` is a tile; any top-level `aggregate:` measures render as KPI cards. For a fixed grid, use `# dashboard { columns=N }` with `# colspan` on each tile (see `skill:malloy-charts`):\n\n```malloy\n# dashboard { columns=2 }\nrun: source -> {\n  nest:\n    # colspan=2\n    # big_value\n    kpis is {\n      aggregate:\n        # label=\"Revenue\"\n        # currency\n        total_revenue\n\n        # label=\"Orders\"\n        # number=auto\n        order_count\n    }\n  nest:\n    # line_chart\n    trend is {\n      group_by: order_date.month\n      aggregate: total_revenue\n      order_by: 1\n    }\n}\n```\n\nKey rendering rules to keep in mind when shaping a cell:\n- FIRST `group_by` = x-axis, FIRST `aggregate` = y-axis.\n- Override field roles with `# x`, `# y`, `# series` on individual fields.\n- For multiple measure series, place `# y` above the `aggregate:` keyword.\n- One aggregate per chart view: use `# dashboard` with nested views for multiple charts.\n- Use `# table.size=fill` for standalone table queries.\n\n## Editing an existing report\n\nFor small targeted changes (fix one cell, insert one new cell), edit that cell in the `.malloynb` file rather than recreating the whole notebook. For structural rewrites (reordering many cells, changing the narrative arc), rewrite the notebook file.\n\n## IMPORTANT\n\nYou CANNOT see the rendered output of notebook cells. Do not claim to see charts, values, or patterns from report cells you haven't explicitly executed via `execute_query`. If you need to analyze results, run the query via `execute_query` first."
     },
     {
       name: "malloy-analyze",
@@ -295639,11 +295697,12 @@ the warning list.
 
 ## Get Diagnostics
 
-**Claude Code (in VS Code terminal):** Call \`mcp__ide__getDiagnostics\` with the file URI.
+Hosts expose errors differently, so take the best path you actually have:
 
-**VS Code Copilot / Cursor:** Use the \`ReadLints\` tool on the file path, or open the file and check lints in the editor.
+1. **An editor-diagnostics tool**, if your host offers one - call it on the file and read the errors straight out.
+2. **Otherwise, compile by running.** Run any query against the source with your query tool and read the error it returns. Every host that can run Malloy can do this, including chat surfaces with no editor.
 
-**Claude Code (standalone terminal):** No IDE diagnostics available. Ask the user to open the file in VS Code with the Malloy extension and report the errors.
+Only ask the user to open the file in an editor when you know they have the model open in one. On a hosted chat surface there is no local checkout to open, and the query path above is the one that works.
 
 ## Strategy
 
@@ -295663,8 +295722,10 @@ the warning list.
 | "Aggregate not allowed in where" | Use \`having:\` instead |
 | 20+ random errors | Backtick reserved word (\`\` \`Date\` \`\`, \`\` \`Hour\` \`\`, \`\` \`number\` \`\`) |
 | \`Can't find field 'X' to set access modifier\` | An \`include {}\` sits before the \`extend { rename: }\`. Rename first, then \`include {}\` naming the field by its new name (see \`skill:malloy-gotchas-modeling\` § Field Management) |
+| \`IO Error: No files found that match the pattern "data/x.csv"\` | Data-file path, not the model. Relative \`duckdb.table()\` paths resolve against the DuckDB \`workingDirectory\`; Publisher sets it to the package root, but a relative \`workingDirectory\` in \`malloy-config.json\` resolves against the process cwd. Make it absolute (see \`skill:malloy-gotchas-modeling\` § Relative Data-File Paths). The "not defined" errors under it are cascade, not real |
 | Import path errors | Check paths: \`import "orders.malloy"\`. All files should be in the same directory (flat layout) |
-| \`from()\` errors | Verify the source query returns the expected columns, check that imported sources are defined |
+| \`unexpected 'from'\` | \`from()\` was removed from the language. Use the query directly: \`source: x is q extend {...}\`, or \`source: x is (q -> {...}) extend {...}\` |
+| Query-based source errors | Verify the source query returns the expected columns, check that imported sources are defined |
 | "Cannot redefine 'X'" | Field already exists from query-based source (\`-> { group_by, aggregate }\`). Remove the dimension, add only NEW derived fields in \`extend {}\`. Use \`include {}\` to add \`#(doc)\` tags to existing fields. |
 
 ## Gotchas Checklist
@@ -295748,7 +295809,7 @@ a / b                           a / nullif(b, 0)
 | "Can't find source X" | Add \`import "X.malloy"\` at top of file (all files in same directory) |
 | Wrong import path | All \`.malloy\` files should be in the package root (flat layout). Use \`import "orders.malloy"\`, not \`import "../sources/orders.malloy"\` |
 | Circular imports | Source A imports Source B which imports Source A. Restructure to break the cycle |
-| \`from()\` "Can't find field" | Verify the source query's GROUP BY and aggregate fields match what you reference in \`extend {}\` |`
+| Query-based source "Can't find field" | Verify the source query's GROUP BY and aggregate fields match what you reference in \`extend {}\` |`
     },
     {
       name: "malloy-define",
@@ -295956,12 +296017,12 @@ A confirmed source architecture and a confirmed set of field definitions (rename
     {
       name: "malloy-getting-started",
       description: "First steps for using a Malloy Publisher deployment through its MCP tools. Use when connecting to Publisher for the first time, when you do not yet know the available environments, packages, or models, or when a user asks what data they can explore. Covers verifying the server, discovering data with malloy_getContext, and running a first grounded query.",
-      body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`. If the point is to author models against a local package, add `--watch-env <env>`: without it Publisher copies local packages at boot and serves the copies, so saved edits are never read.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nTwo escape hatches worth knowing:\n\n- **When the session cannot be relaunched from the workspace directory** (a project `.mcp.json` is only discovered by sessions that *start* in its directory), register the server at user scope so the directory stops mattering: `claude mcp add --transport http malloy http://localhost:4040/mcp -s user` (use the MCP port the server actually bound; its startup log prints it). Caveat: for sessions that do start in the workspace, the project `.mcp.json` shadows the user-scoped entry, so prefer the project file when it is discoverable.\n- **Do not trust an existing `.mcp.json`\'s URL blindly.** The file outlives the server that wrote it, and a boot that failed partway (for example, the REST port was taken) can leave it pointing at a dead port while a live server sits on another. If connecting fails or answers look wrong, confirm identity with `malloy_getContext`, which names the environment and packages you are really talking to; that check works on every platform, which the port check does not (`lsof -iTCP:4040 -sTCP:LISTEN` on macOS and Linux, `netstat -ano | findstr :4040` on Windows).\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\nIf the data you want is in a connected database but not yet in any package, use `malloy_searchDatabaseSchema` instead of `malloy_getContext`: it walks a connection\'s schemas and tables and ranks them against a plain-English description, and hands back the `source:` line to start a model from. It returns names and types only, so to see what a column actually contains run `malloy_executeQuery` against a model in a package that uses the same connection, with an ad-hoc query like `run: my_conn.table(\'sales.orders\') -> { group_by: order_status }`. That tool needs an existing model to run against, so a table you have not modelled yet has none of its own.\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.'
+      body: '# Getting started with Malloy Publisher\n\nGoal: go from "connected" to a correct, grounded answer without guessing any names.\n\n## 0. Confirm the tools are reachable\n\nAt minimum you need `malloy_getContext`, `malloy_executeQuery`, and `malloy_searchDocs`. Authoring a model also needs `malloy_compile` and `malloy_reloadPackage` (see section 4); an older Publisher may not serve those two.\n\nIf none of the tools are there, either the server is not running or your client connected before it was. Start the server (`npx @malloy-publisher/server --port 4000`, or `bun run build && bun run start` from a clone) and wait until `curl -s http://localhost:4000/api/v0/status` reports `operationalState: serving`. If the point is to author models against a local package, add `--watch-env <env>`: without it Publisher copies local packages at boot and serves the copies, so saved edits are never read.\n\nIf there is no Publisher workspace here at all, and the user wants to work with data of their own rather than the bundled examples, `npm create @malloy-publisher/malloy-package@latest <name>` scaffolds one: the package and a starter model, registered so the server actually serves it, plus the start script, the MCP config and these skills. Keep the `@latest` when you type it: `npm create` resolves through npm\'s npx cache and an unversioned name is satisfied by any copy already there, so on a machine that has scaffolded before npm never asks the registry and you get an old scaffolder pinning an old server, with nothing to say so. Run bare, it comes with a small sample dataset, so there is something to query straight away. In a fresh directory `npm start` then runs the pinned server against the package in watch mode; if the directory already had a `package.json` the scaffolder leaves it alone and adds no script, printing the equivalent `npx` command to use instead. Where you run it matters: only the package lands in `<name>/`, and the workspace files, the agent instructions and the MCP config among them, are written to the current directory. Run it here if this directory is empty or is meant to become the workspace. If it already holds other work, scaffold into a new directory instead (`mkdir my-data && cd my-data`), because agent config is discovered by walking up, so writing those files here changes what every session beneath this directory inherits. Seed the starter model from a local file with `npm create @malloy-publisher/malloy-package@latest <name> -- --data <path/to/their-file.csv>` (CSV, Parquet, or Excel `.xlsx`), keeping the `--`, which is how `npm create` passes options through. That path is relative to wherever you run the command, so if you scaffolded into a new directory it has to reach back out to their file; the scaffolder copies it into the package and leaves the original alone. A seeded package starts smaller than the sample one, since the scaffolder does not read their columns: expect a row count and an overview, and build the model from there. A package is just Malloy, so it can instead query a database connection the config defines. Because it writes a `.mcp.json` that did not exist when the client connected, the user has to restart or reconnect once before these tools appear, and their client will ask them to approve the new project-scoped server the first time. That only works when the workspace is at the session\'s own root, so if you scaffolded into a new directory below that root, the user has to open a session there instead: a `.mcp.json` further down is never discovered.\n\nIf you started the server yourself in this session, the tools still will not appear: your tool list was fixed when you connected, and you cannot reconnect yourself. Tell the user the tools are missing for that reason and ask them to run `/mcp`, select `malloy`, and choose Reconnect. The panel offers `Authenticate` first and reports `Auth: not authenticated`; that is a red herring, the endpoint has no auth. Restarting Claude Code also works. Continue once the tools are there.\n\nTwo escape hatches worth knowing:\n\n- **When the session cannot be relaunched from the workspace directory** (a project `.mcp.json` is only discovered by sessions that *start* in its directory), register the server at user scope so the directory stops mattering: `claude mcp add --transport http malloy http://localhost:4040/mcp -s user` (use the MCP port the server actually bound; its startup log prints it). Caveat: for sessions that do start in the workspace, the project `.mcp.json` shadows the user-scoped entry, so prefer the project file when it is discoverable.\n- **Do not trust an existing `.mcp.json`\'s URL blindly.** The file outlives the server that wrote it, and a boot that failed partway (for example, the REST port was taken) can leave it pointing at a dead port while a live server sits on another. If connecting fails or answers look wrong, confirm identity with `malloy_getContext`, which names the environment and packages you are really talking to; that check works on every platform, which the port check does not (`lsof -iTCP:4040 -sTCP:LISTEN` on macOS and Linux, `netstat -ano | findstr :4040` on Windows).\n\nWhen a user is present, do not route around it by calling the REST API with curl. It appears to work, so the user never learns their session is missing the tools, and you lose what they are for: grounded discovery instead of guessed names, `malloy_compile` instead of throwaway queries, and `malloy_reloadPackage` instead of a restart. Say the tools are missing and let the user fix it in five seconds. Running unattended, with nobody who can reconnect you, is different: there the REST API is the supported interface, not a workaround. Discovery, query, compile, and reload all have REST equivalents (`malloy_searchDocs` and `malloy_getContext`\'s plain-English ranking do not; read the bundled skills for syntax and ground from model metadata instead); the running server serves the full spec at `http://localhost:4000/api-doc.yaml`, and AGENTS.md carries the endpoint map.\n\n## 0.5 Ask whether they also use the CLI or the VS Code extension\n\nPublisher is often not the only thing reading the model. The Malloy CLI (`malloy-cli`) and the VS\nCode extension compile the same files, and they do **not** get their connections from\n`publisher.config.json` - they read `malloy-config.json`, found by walking up from the file being\ncompiled. So a package that Publisher serves correctly can fail to compile in the editor, on the\nsame machine, from the same files.\n\nAsk before the user finds out the hard way:\n\n> Are you also using the Malloy CLI or the VS Code extension on this package?\n\nIf yes, and the package reads **local data files**, it needs a `malloy-config.json`. Publisher\nresolves a relative `duckdb.table(\'data/x.csv\')` against the package root on its own, so nothing is\nconfigured for it; the other two hosts resolve it against the DuckDB connection\'s\n`workingDirectory`, which has to be set.\n\n**Make that path absolute.** A relative `workingDirectory` is resolved against the process\'s current\ndirectory, not the config file\'s directory and not the VS Code workspace root, so the same config\ncompiles from one directory and fails from another:\n\n```json\n// malloy-config.json, beside the model\n{\n  "connections": {\n    "duckdb": {\n      "is": "duckdb",\n      "workingDirectory": "/abs/path/to/package",\n      "securityPolicy": "none"\n    }\n  }\n}\n```\n\nPoint it at the directory the model\'s table paths are written relative to - the one holding `data/`.\nLeave the model\'s own paths relative so Publisher still serves it; only the config carries the\nabsolute path. When this is wrong the editor reports `IO Error: No files found that match the\npattern "data/x.csv"` on the `source:` line, followed by a "not defined" error for every field of\nthat source; those are cascade, not real. The `malloy-gotchas-modeling` skill covers the mechanism\nunder § Relative Data-File Paths.\n\n**Remote Credible connections are a different case.** With the Credible extension running and signed\nin, the data is reached through a `publisher` proxy connection - the connection type that forwards\nSQL to a remote Publisher dataplane, and the same type the CLI and the VS Code extensions use - and\nthe extension supplies it, so there is nothing to hand-write and no `workingDirectory` involved.\n`workingDirectory` only ever matters for local files that DuckDB opens itself. Worth knowing: those\naccess tokens are user-scoped and short-lived, and nothing refreshes them mid-session, so queries\nthat start failing auth after a long session mean the token expired, not that the model broke. See\n`docs/connections.md` § Publisher proxy connections.\n\n## 1. Discover what exists (never guess names)\n\n`malloy_getContext` is progressive. Call it with as much as you know:\n\n- No arguments: the available environments, each with its package names.\n- `environmentName` only: the packages in that environment.\n- `environmentName` + `packageName`: that package\'s sources.\n- `environmentName` + `packageName` + `query` (plain English): the sources, views, named queries, and dimension/measure fields most relevant to the question.\n\nUse the names it returns exactly. Do not invent environments, packages, sources, or fields.\n\n## 2. Run the query\n\nCall `malloy_executeQuery` with the `environmentName`, `packageName`, and `modelPath` from the context results, plus either:\n\n- a named view or query: pass its `name` as `queryName` (with `sourceName` for a view), or\n- an ad-hoc query: pass Malloy code as `query`.\n\nThe result is JSON. Charts and dashboards defined in the model render in the Publisher UI at http://localhost:4000.\n\n## 3. When you need Malloy syntax\n\nUse `malloy_searchDocs` for language questions (filters, aggregates, joins, nesting, renderers).\n\nIf the data you want is in a connected database but not yet in any package, use `malloy_searchDatabaseSchema` instead of `malloy_getContext`: it walks a connection\'s schemas and tables and ranks them against a plain-English description, and hands back the `source:` line to start a model from. It returns names and types only, so to see what a column actually contains run `malloy_executeQuery` against a model in a package that uses the same connection, with an ad-hoc query like `run: my_conn.table(\'sales.orders\') -> { group_by: order_status }`. That tool needs an existing model to run against, so a table you have not modelled yet has none of its own.\n\n## 4. What else you can do here\n\nAnswering questions is the start, not the whole surface. When the user asks what is possible, say so rather than offering queries alone. Switch skills for the deeper work:\n\n- `malloy-modeling`: build or change a model. Validate the edit with `malloy_compile`, save it, then `malloy_reloadPackage` so the new sources and views run by name without restarting the server.\n- `malloy-analysis`: explore a package and answer data questions.\n- `malloy-html-data-apps`: build a data app, a hand-authored HTML page in the package\'s `public/` directory that Publisher serves, backed by the package\'s models and needing no build step.\n- `malloy-review`: check Malloy for correctness.\n\n## Contract\n\n- Ground every query in `malloy_getContext` results. If a name is not in the results, do not use it.\n- Start broad and narrow down: environments, then packages, then sources, then query.\n- Confirm the environment and package before running a query.'
     },
     {
       name: "malloy-gotchas-modeling",
       description: "Common Malloy modeling mistakes and how to avoid them. Read BEFORE writing source definitions, dimensions, measures, or joins. Covers reserved words, NULL checks, date functions, type casts, field management (extend except/accept/rename vs include public/internal/private), and query-based source gotchas.",
-      body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: `unit(start to end)`, and the unit decides the operand type\n\nAn interval is `unit(start to end)`. Two rules, both enforced by the compiler:\n\n- **Never subtract.** `days(a - b)` fails with `Can not offset time by 'date'`. The `to` form is the only one.\n- **Mixing a date and a timestamp needs a cast, unless the date side is a literal.** A date *literal* widens to a timestamp on its own, so `days(@2020-01-01 to now)` compiles. A date *column* does not: `days(signup_date to now)` fails with `Cannot measure from date to timestamp`. Cast the odd one out (`::date`, `::timestamp`). `now` is a timestamp.\n\nWhich units accept what:\n\n| Units | Kind | Operands |\n|-------|------|----------|\n| `seconds`, `minutes`, `hours`, `days` | clock | timestamps or dates |\n| `weeks`, `months`, `quarters`, `years` | calendar | **dates only**: on timestamps they fail with `Cannot measure interval using 'month' for 'timestamp' values; calendar interval measurement requires dates` |\n\n```malloy\n// WRONG: subtraction, and a calendar unit applied to timestamp columns\ndimension: gap is days(closed_at - opened_at)\ndimension: months_open is months(opened_at to closed_at)\n\n// WRONG: approximating a calendar unit that exists\ndimension: months_open is days(opened_at to closed_at) / 30.44\n\n// RIGHT\ndimension: days_open   is days(opened_at to closed_at)\ndimension: months_open is months(opened_at::date to closed_at::date)\n```\n\nThe calendar units are real and exact. If one fails, read the message: it is telling you to cast the operands, not to divide by 30.44.\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` and `include {}`, in that order\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` handles the renames.** They do compose, but only in one order: the `extend {}` that renames must come **before** the `include {}`, and `include {}` must name the field as it is *after* the rename.\n\n| Mechanism | Where it lives | Keywords | Experimental flag? |\n|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | Yes (`##! experimental.access_modifiers`) |\n| Field management | `extend {}` | `accept:` / `except:` / `rename:` | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When a `rename:` is needed: rename first, then `include {}`\n\nThe usual reason is a collision inside `include {}`: a measure cannot share a name with a raw column, even one tagged `internal:`, and the compiler says so (`Cannot redefine 'revenue' 'revenue' is internal`). The fix is to rename the raw column out of the way, which frees the name for the measure. Order is what makes it work:\n\n```malloy\n##! experimental.access_modifiers\n// RIGHT: rename frees `revenue`, include curates what is left, measure takes the name\nsource: orders is conn.table('orders')\n  extend { rename: raw_revenue is revenue }\n  include {\n    #(doc) Revenue as loaded, before adjustments\n    internal: raw_revenue\n    public: order_id, user_id\n  }\n  extend { measure: revenue is raw_revenue.sum() }\n```\n\nTwo ways to get the order wrong, with the errors they produce:\n\n- **`include {}` before the renaming `extend {}`** fails with `Can't find field 'X' to set access modifier`, currently surfaced as an internal compiler error. `include` runs against names that no longer exist by the time the rename is applied.\n- **Naming the pre-rename column inside `include {}`** fails with `` `revenue` not found ``. After a rename only the new name exists; use it.\n\nYou do not have to give up `include {}` to get a rename: the curated surface, `#(doc)` on raw columns, and the `public/internal/private` tiers all survive. Renaming the *measure* instead is still worth considering when the raw column name is the one people know, but it is a modeling preference, not a workaround for a limitation.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: `include { internal: ... }` (lets you also `#(doc)` the public columns). A `rename:` in the same source does not force you off `include {}` - see item 4 for the order.\n4. SQL aliases: an `extend { rename: ... }` before `include {}`, naming the field by its new name in `include {}` (they compose, but only in that order). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## Mixed-Grain Joins: A Pre-Aggregated Source Ignores Your Filters\n\nJoining an aggregate-grain source (a decade/month/region summary table) into a detail-grain source produces values that do **not** respond to the query's filters. Malloy's symmetric aggregates prevent fan-out; they cannot prevent this, because the joined value is unfiltered *by construction*: it was computed over the whole population before the query ran.\n\n```\nrun: track_analysis -> {\n  where: genre = 'Rock'\n  group_by: decade\n  aggregate: track_count                    // filtered: Rock only     -> 701\n  group_by: decade_trends.decade_track_count // unfiltered population  -> 1,088\n}\n```\n\nTwo count-shaped numbers side by side, one filtered and one not; read as \"701 of 1,088 Rock tracks\" it is simply wrong: 1,088 is every genre. Two legitimate resolutions:\n\n- **Keep the join as a population baseline** when comparing a row to the whole population is the intent (e.g. `energy_vs_decade`). Then every joined field's `#(doc)` must say it is a fixed population value that does not respond to filters, and count-shaped fields with no comparison purpose (like `decade_track_count`) should be `internal:`; they only invite the misreading.\n- **Compute the aggregate as a query-based source from the detail table** so it derives from one source of truth and the derivation is visible.\n\nThis is the modeling-time consequence of ignoring `skill:malloy-scope`'s advice to skip pre-aggregated snapshot tables and compute fresh in Malloy instead.\n\n## Thresholds Are Decisions, Not Syntax\n\nBefore writing a `pick` expression or filtered measure with a numeric cutoff, see `skill:malloy-model` § Key Rules: every boundary must be user-supplied, distribution-derived (query the percentiles first), or explicitly flagged as an assumption in its `#(doc)`. Never invent one silently.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `months()`): always `unit(start to end)`, and calendar units need date operands (see above)\n- Query-based sources (`from()`)\n- `!` operator / `sql_number()`"
+      body: "# Modeling Gotchas\n\n> **Read this before writing Malloy code.** These patterns cause most modeling errors.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Reserved Words: Backtick Them\n\n**When in doubt, backtick it.** Unquoted reserved words cause cascading errors on unrelated lines.\n\n```malloy\n// WRONG                        // RIGHT\ndimension: d is Date::date      dimension: d is `Date`::date\n```\n\nWords most likely to appear as column names:\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n\n## NULL Checks: `is not null`, NOT `!= null`\n\n```malloy\n// WRONG                             // RIGHT\ndimension: is_sold is sold_at != null   dimension: is_sold is sold_at is not null\n```\n\n## Date Functions vs Properties\n\n```malloy\n// WRONG: day_of_week is a function        // RIGHT\ndimension: dow is created_at.day_of_week   dimension: dow is day_of_week(created_at)\n```\n\n**Property access:** `.month`, `.year`, `.quarter`, `.day`, `::date`\n**Function call required:** `day_of_week()`, `week()`, `hour()`, `minute()`, `second()`\n\n## `.date` Is a Cast, Not a Truncation\n\nCalendar truncations are `.day`, `.week`, `.month`, `.quarter`, `.year` (plus `.hour`, `.minute`, `.second` for timestamps). `.date` is **not** among them: it's a **cast** (`::date`), not a truncation, so `created_at.date` does not compile. This bites twice: once at compile time, and again as a latent bad `#(doc)` comment that only a review pass catches (\"truncated to date\" is a doc smell; it should say \"to day\").\n\n```malloy\n// WRONG                          // RIGHT\ncreated_at.date                   created_at.day     // truncate to day\n                                   created_at::date   // cast to a date\n```\n\n## Interval Functions: `unit(start to end)`, and the unit decides the operand type\n\nAn interval is `unit(start to end)`. Two rules, both enforced by the compiler:\n\n- **Never subtract.** `days(a - b)` fails with `Can not offset time by 'date'`. The `to` form is the only one.\n- **Mixing a date and a timestamp needs a cast, unless the date side is a literal.** A date *literal* widens to a timestamp on its own, so `days(@2020-01-01 to now)` compiles. A date *column* does not: `days(signup_date to now)` fails with `Cannot measure from date to timestamp`. Cast the odd one out (`::date`, `::timestamp`). `now` is a timestamp.\n\nWhich units accept what:\n\n| Units | Kind | Operands |\n|-------|------|----------|\n| `seconds`, `minutes`, `hours`, `days` | clock | timestamps or dates |\n| `weeks`, `months`, `quarters`, `years` | calendar | **dates only**: on timestamps they fail with `Cannot measure interval using 'month' for 'timestamp' values; calendar interval measurement requires dates` |\n\n```malloy\n// WRONG: subtraction, and a calendar unit applied to timestamp columns\ndimension: gap is days(closed_at - opened_at)\ndimension: months_open is months(opened_at to closed_at)\n\n// WRONG: approximating a calendar unit that exists\ndimension: months_open is days(opened_at to closed_at) / 30.44\n\n// RIGHT\ndimension: days_open   is days(opened_at to closed_at)\ndimension: months_open is months(opened_at::date to closed_at::date)\n```\n\nThe calendar units are real and exact. If one fails, read the message: it is telling you to cast the operands, not to divide by 30.44.\n\n## Safe Division: Always `nullif`\n\n```malloy\n// WRONG              // RIGHT\na / b                 a / nullif(b, 0)\n```\n\n## String Columns Need Casts for Aggregates\n\n```malloy\n// WRONG: \"Can't use type string\"     // RIGHT\nmeasure: avg_score is avg(score)      measure: avg_score is avg(score::number)\n```\n\n**Dirty columns: null the sentinel before casting.** `::number` is a strict cast, so a column that carries non-numeric sentinels (`'NA'`, `'N/A'`, `''`, `'-'`, `'null'`) compiles fine but fails at query time with `Could not convert string 'NA' to DOUBLE`. Strip the sentinel with `nullif` first, then cast (aggregates skip nulls):\n\n```malloy\n// WRONG: throws on 'NA' at query time   // RIGHT: nulls 'NA', then casts\nmeasure: s is avg(score::number)         measure: s is avg(nullif(score, 'NA')::number)\n```\n\nChain `nullif` for multiple sentinels: `nullif(nullif(score, 'NA'), '')::number`. Sample the column's values first (`run: source -> { group_by: score; limit: 20 }`) to see which sentinels it uses.\n\n## Boolean Columns: No Quotes\n\n```malloy\n// WRONG                                     // RIGHT\ncount() { where: complaint = 'true' }        count() { where: complaint = true }\n```\n\nCheck schema: if `BOOL`, use `true`/`false`. If `STRING`, use `'true'`/`'false'`.\n\n## `greatest()` / `least()` Are Null-Poisoning\n\nMalloy's `greatest()` / `least()` return **NULL if *any* argument is null**, unlike Postgres `GREATEST`/`LEAST`, which ignore nulls. Porting a LookML/SQL expression verbatim is a silent parity bug: the number just goes null for any row with a missing input. Coalesce the result back to a non-null argument:\n\n```malloy\n// WRONG: one null input nulls the whole thing\ndimension: last_touch is greatest(email_at, call_at)\n\n// RIGHT: fall back so a null arg can't poison the result\ndimension: last_touch is greatest(email_at, call_at) ?? email_at ?? call_at\n```\n\n## No Scalar Median; Raw-SQL Aggregates Don't Compile\n\n**There is no scalar `median`, and `PERCENTILE_CONT` cannot be expressed as a measure in this build.** Every documented form for a custom SQL aggregate - `percentile_cont!(x, 0.5)`, `sql_number(...)`, `sql_number(...) { is_aggregate: true }`, and the `# is_aggregate` annotation - resolves as a **scalar** and fails with *\"Cannot use a scalar field in a measure declaration.\"* The docs' own `avg_dist` example fails the same way. This is a deployed-runtime limitation, not a syntax error you can fix: **do not** burn cycles trying `!`, `sql_number`, or `is_aggregate` variations to get a median.\n\n```malloy\n// DOES NOT COMPILE in this build (all forms resolve as scalar):\nmeasure: median_x is percentile_cont!(x, 0.5)\nmeasure: median_x is sql_number(\"PERCENTILE_CONT(...) ...\") { is_aggregate: true }\n```\n\n**Ship `avg` instead, or defer median with a documented gap** (\"median deferred: no scalar median / runtime rejects raw-SQL aggregates\"). Tell the user; don't silently substitute `avg` for a metric that was specified as median.\n\n**`stddev` does work**, so reach for it when the question is about spread. It is a native Malloy aggregate rather than a raw-SQL escape, so unlike everything above it compiles both inline and as a `measure:`, and it is the sample standard deviation. `variance`, `stddev_samp`, and `stddev_pop` are not Malloy functions, and pushing them through `!` fails as a scalar exactly like `percentile_cont!`.\n\n```malloy\n// WORKS: inline, or as a measure on a source\nrun: order_items -> { aggregate: sd is stddev(sale_price) }\nsource: items is order_items extend { measure: price_stddev is stddev(sale_price) }\n```\n\n## Field Management: `extend {}` and `include {}`, in that order\n\nMalloy has two field-management mechanisms for base sources. **`include {}` is the curated default; `extend { except / accept / rename }` handles the renames.** They do compose, but only in one order: the `extend {}` that renames must come **before** the `include {}`, and `include {}` must name the field as it is *after* the rename.\n\n| Mechanism | Where it lives | Keywords | Experimental flag? |\n|---|---|---|---|\n| Access modifiers (default) | `include {}` | `public:` / `internal:` / `private:` | Yes (`##! experimental.access_modifiers`) |\n| Field management | `extend {}` | `accept:` / `except:` / `rename:` | No |\n\n### Default: `include {}` for documented, curated base sources\n\nUse `include {}` whenever the source doesn't need a `rename:`. It's the only way to attach `#(doc)` tags to raw columns, and it's the canonical way to hide empty/garbage/duplicate columns (`internal:`) and sensitive ones (`private:`). See `skill:malloy-model` § Access Modifiers.\n\n```malloy\n##! experimental.access_modifiers\nsource: orders is conn.table('orders') include {\n  public:\n    #(doc) Order identifier\n    order_id\n\n    #(doc) Customer who placed the order\n    user_id\n\n  internal:\n    raw_payload_json    // empty after JSON extraction\n    legacy_status_code  // superseded by status_code\n}\n```\n\n### When a `rename:` is needed: rename first, then `include {}`\n\nThe usual reason is a collision inside `include {}`: a measure cannot share a name with a raw column, even one tagged `internal:`, and the compiler says so (`Cannot redefine 'revenue' 'revenue' is internal`). The fix is to rename the raw column out of the way, which frees the name for the measure. Order is what makes it work:\n\n```malloy\n##! experimental.access_modifiers\n// RIGHT: rename frees `revenue`, include curates what is left, measure takes the name\nsource: orders is conn.table('orders')\n  extend { rename: raw_revenue is revenue }\n  include {\n    #(doc) Revenue as loaded, before adjustments\n    internal: raw_revenue\n    public: order_id, user_id\n  }\n  extend { measure: revenue is raw_revenue.sum() }\n```\n\nTwo ways to get the order wrong, with the errors they produce:\n\n- **`include {}` before the renaming `extend {}`** fails with `Can't find field 'X' to set access modifier`, currently surfaced as an internal compiler error. `include` runs against names that no longer exist by the time the rename is applied.\n- **Naming the pre-rename column inside `include {}`** fails with `` `revenue` not found ``. After a rename only the new name exists; use it.\n\nYou do not have to give up `include {}` to get a rename: the curated surface, `#(doc)` on raw columns, and the `public/internal/private` tiers all survive. Renaming the *measure* instead is still worth considering when the raw column name is the one people know, but it is a modeling preference, not a workaround for a limitation.\n\n### `extend {}` clauses (reference)\n\n- **`accept:`**: allow-list, keep only the named columns\n- **`except:`**: deny-list, drop the named columns; keep everything else (mutually exclusive with `accept:`)\n- **`rename:`**: alias a raw column to free up its original name for a measure or dimension\n\n### Migrating `conn.sql()` to `conn.table()` + Malloy clauses\n\nThe biggest reason teams reach for `conn.sql()` is column gating, aliasing, and per-row derivation in one place. All three have native equivalents:\n\n1. **Verify the schema**: `run: <source> -> { select: *; limit: 1 }` to discover all columns. Anything in the table but not in the SQL's `SELECT` was being intentionally hidden, so preserve that gating.\n2. Switch to `conn.table('…')`.\n3. Hidden columns: `include { internal: ... }` (lets you also `#(doc)` the public columns). A `rename:` in the same source does not force you off `include {}` - see item 4 for the order.\n4. SQL aliases: an `extend { rename: ... }` before `include {}`, naming the field by its new name in `include {}` (they compose, but only in that order). If the alias was to free up a name for a measure, use `rename: raw_X is X`, then `measure: X is raw_X.sum()`.\n5. SQL derivations: `dimension:` definitions in `extend {}`.\n6. SQL `WHERE`: source-level `where:`.\n\n## Cannot Redefine Query-Based Source Columns\n\nColumns from `table -> { group_by, aggregate }` or `conn.sql()` already exist. You cannot re-declare them.\n\n```malloy\n// WRONG: \"Cannot redefine 'user_id'\"\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: user_id is user_id }\n// RIGHT: add only NEW derived dimensions\nsource: facts is conn.table('t') -> { group_by: user_id, aggregate: total is sum(amt) }\n  extend { dimension: is_high_value is total > 1000 }\n```\n\nTo add `#(doc)` tags to existing query columns, use `include {}` between the query and extend.\n\n## Extending a Source Cannot Reuse a Name It Already Defines\n\n```malloy\n// WRONG: \"Cannot redefine 'overview'\" when sales already declares view: overview\nsource: wines is sales extend { view: overview is { aggregate: record_count } }\n// RIGHT: give the extension its own name\nsource: wines is sales extend { view: summary is { aggregate: record_count } }\n```\n\nAn extension adds to the parent's namespace, it does not override it. This bites when you extend a source to \"replace\" one of its views: rename the new definition, or edit the view on the parent source instead of extending it. Malloy reports the same `Cannot redefine 'X'` for dimensions and measures that collide with an inherited name, per the sections above and below.\n\n## Never Use `conn.sql()` When Malloy Has a Native Pattern\n\n```malloy\n// WRONG: raw SQL for pre-aggregation\nsource: facts is conn.sql(\"\"\"SELECT user_id, SUM(amount) AS total FROM orders GROUP BY user_id\"\"\")\n// RIGHT: Malloy query-based source\nsource: facts is conn.table('orders') -> { group_by: user_id, aggregate: total is sum(amount) }\n```\n\n**Mandatory: call `search_malloy_docs` before reaching for `conn.sql()`.** Don't argue from intuition. Most patterns that look SQL-only have a Malloy equivalent, including the ones reviewers historically said couldn't be expressed.\n\n| Looks like it needs SQL | Malloy equivalent |\n|---|---|\n| Multi-CTE pipeline | Stacked query-based sources: `source: a is t -> {...}`; `source: b is a -> {...}`; `source: c is b -> {...}` |\n| UNNEST / array column access | `array_column.each.field`: arrays auto-join as nested tables ([data types docs](https://docs.malloydata.dev/documentation/language/datatypes#array-access)) |\n| PIVOT (conditional aggregation) | Filtered aggregates: `aggregate: a is x.sum() { where: cat = 'a' }, b is x.sum() { where: cat = 'b' }` |\n| Window functions (any frame, including custom) | `calculate:` with `sum_cumulative`, `lag`, `lead`, `rank`, `row_number`, `avg_moving`, `first_value`, `last_value`: supports `partition_by:` and `order_by:` ([window functions docs](https://docs.malloydata.dev/documentation/language/functions#window-functions)) |\n| `ROWS BETWEEN UNBOUNDED PRECEDING AND 1 PRECEDING` | `sum_cumulative(x) - x` (cumulative-including-current minus current = cumulative-excluding-current) |\n| `WHERE date = (SELECT max(date) FROM …)` (latest snapshot) | `join_cross` to a one-row aggregate source, then filter on the joined `max_date` field |\n| Multi-key joins | `join_one: x is target on a = x.a and b = x.b and c = x.c` |\n| `greatest()` / `least()` / `CASE` chains | All native: `greatest(a, b, c)`, `least(a, b)`, `pick 'x' when cond else 'y'` |\n| Dialect-specific scalar functions | `function_name!return_type(args)`: Malloy's raw-SQL function escape (no `conn.sql()` block needed) |\n\n**Genuinely valid `conn.sql()` candidates (rare):**\n\n- SQL features Malloy explicitly doesn't model (e.g., DML/DDL, specific `MERGE` patterns)\n- Multi-stage transformations where every CTE has 3+ joins to different tables AND the result is consumed by multiple downstream sources, but in this case an intermediate table in the data warehouse is usually still better than `conn.sql()`\n\n**Never use `conn.sql()` for:** simple column selection or renaming, `WHERE` filters, two-table joins, column type casts, latest-snapshot patterns, conditional aggregation, or window functions of any kind.\n\nIf a project's standards file specifies a stricter policy (e.g., a `search_malloy_docs` rationale comment requirement above every `conn.sql()` block), defer to that.\n\n## Relative Data-File Paths: Set an Absolute `workingDirectory`\n\n`duckdb.table('data/x.csv')` is resolved against the DuckDB connection's `workingDirectory`, not\nagainst the model file. Publisher sets that to the package root, so relative paths work there with\nno config at all. Every other host reads it from `malloy-config.json`, and **a relative value there\nis resolved against the process's current directory** - not the config file's directory, and not the\nVS Code workspace root. `canonicalizeConfigPath` in `malloy-db-duckdb/src/duckdb_config.ts` calls\n`canonicalizePath` with no `baseDirectory`, which is a bare `path.resolve(input)`.\n\nSo the same config works or fails depending on which directory the editor or shell was launched\nfrom. That is why this breaks intermittently and appears to be a model bug.\n\n```json\n// WRONG: resolved against the process cwd, so it works from the repo root and\n// fails from anywhere else - including however your editor happened to launch\n{\"connections\": {\"duckdb\": {\"is\": \"duckdb\", \"workingDirectory\": \"malloy\"}}}\n\n// RIGHT: absolute, so the cwd cannot change the answer\n{\"connections\": {\"duckdb\": {\"is\": \"duckdb\", \"workingDirectory\": \"/abs/path/to/pkg\"}}}\n```\n\nPoint it at the directory the model's table paths are written relative to - the package root, the\none holding `data/`. Keep the model's paths relative (`data/x.csv`) so Publisher still serves it;\nonly the config carries the absolute path.\n\n**The symptom, and the cascade.** One `IO Error` at the `source:` line, then a \"not defined\" error\nfor every field of that source:\n\n```\nline 87: IO Error: No files found that match the pattern \"data/product_usage.csv\"\nline 91: 'org_slug' is not defined\nline 93: Reference to undefined value active_users\n```\n\nThose field errors are not real. Fix the first error and they all go. Do not start renaming\ncolumns.\n\n**Check the config before the model** when a source that Publisher queries fine fails in the editor\nor CLI with a missing-file error. The model is the same file; the resolution base is not.\n\n## JSON Files: Read Them In Place Like CSV\n\n```malloy\n// RIGHT: .json works like .csv/.parquet\nsource: reviews is duckdb.table('data/reviews.json')\n// RIGHT: newline-delimited JSON is read the same way\nsource: events is duckdb.table('data/events.ndjson')\n// RIGHT: read options need read_json_auto in a SQL source\nsource: nested is duckdb.sql(\"\"\"SELECT * FROM read_json_auto('data/reviews.json')\"\"\")\n// WRONG: shelling out to python, or converting to CSV first\n```\n\nDuckDB reads JSON directly, so never preprocess a `.json` file before modeling it and never reach for a scripting language to inspect one. Both a top-level array of objects and newline-delimited JSON work through `duckdb.table()`.\n\nQuirk: JSON carries no schema, so a value written as `\"90\"` arrives as a string where the same data in CSV would be inferred as a number. Cast it in the source, under a new name (reusing the column's own name is a redefinition error):\n\n```malloy\nsource: reviews is duckdb.table('data/reviews.json') extend {\n  dimension: points_num is points::number\n}\n```\n\n## Excel Files: Read `.xlsx` In Place, Never Convert\n\n```malloy\n// RIGHT when the sheet is a plain table (header in row 1, data under it, no blank row inside\n// it): read it where it sits, like .csv/.parquet (in a Publisher package the sandbox\n// connection is `duckdb`)\nsource: budget is duckdb.table('data/budget.xlsx')\n// RIGHT for anything messier. Profile the top rows first to find the real header row and the\n// last real column, because nothing else will tell you where they are. Put the probe in the\n// model file as its own source: Publisher refuses raw SQL in an ad-hoc query.\n//   SELECT * FROM read_xlsx('data/sales.xlsx', sheet = 'Sales Data',\n//                           range = 'A1:Z15', header = false, all_varchar = true)\nsource: sales is duckdb.sql(\"\"\"\n  SELECT * FROM read_xlsx('data/sales.xlsx',\n    sheet = 'Sales Data',      -- EDIT: only the first sheet is read by default\n    header = true,\n    range = 'A5:J100000'       -- EDIT: A5 is the real header row. Keep the column bound at the\n  )                            -- last real column; the row bound just has to clear the end.\n  WHERE \"Order ID\" LIKE 'SO-%' -- EDIT, REQUIRED: a data-row predicate. This is what ends the\n\"\"\")                           -- read; drop it and every empty row in the range comes back.\n// WRONG: converting the spreadsheet to Parquet or CSV first (an unnecessary extra step)\n```\n\nDo not convert spreadsheets before modeling. DuckDB's excel extension reads `.xlsx` directly and loads automatically on first use, so a sheet that is a plain table needs nothing more than `duckdb.table()`. Converting does not avoid any of the problems below, it just moves them into a copy that goes stale the next time someone updates the workbook.\n\n**Plenty of real exports are not plain tables, and nothing tells you.** A report title, a \"generated on\" banner, a merged group header, a blank line above the header, or a blank spacer row inside the data are all ordinary, and none of them is visible from Malloy. There is no error either: the package loads, the server reports serving, the query returns 200, and the number is just wrong. So make two checks before building on the read: compare `aggregate: record_count is count()` against what you know is in the file, and `select: *; limit: 1` to see what the columns really are. If either disagrees with the file, the read is wrong and so is every measure over it.\n\n`table()` takes a plain file path only, so anything needing `read_xlsx` options (`sheet`, `range`, `header`, `ignore_errors`, `normalize_names`, `all_varchar`, `empty_as_varchar`, `stop_at_empty`) goes through the SQL-source form.\n\nQuirks:\n\n- Only the FIRST sheet is read by default. Select another with `sheet = 'Name'`. There is no function that lists a workbook's sheet names, but passing one that does not exist reports a suggestion (`Sheet \"x\" not found ... Did you mean: \"Notes\"`), which is one way to find a name you were not given.\n- A title or banner row above the header collapses the read. DuckDB takes the first row it finds as the column names, so a lone title cell in A1 becomes the only column. How many rows you then get is the next quirk's business: whatever sits between the title and the first blank row, often none or one, otherwise a plausible-looking partial count. Pass a `range` that starts at the real header row.\n- With no `range`, `stop_at_empty` defaults to true and the read stops at the first blank row, which on a real sheet is usually a spacer between blocks rather than the end of the data: a 30-row sheet with one spacer after row 10 reads as 10 rows. `stop_at_empty = false` lifts that, but it only helps when the header really is in row 1; with a title above the header you need the `range` anyway, and a `range` flips the default for you. It also hands the blank rows back as all-null rows, so the count comes out one high per spacer until you filter them.\n- A `range` reads every cell inside it, so an overshot bound manufactures padding: past the last real column you get all-null fields (`A5:Z100000` on a ten-column sheet yields 26, the extras named `C10` and `_1` through `_15`), and past the last real row all-null rows (`A5:J100000` on a 1,500-row sheet reads 99,995). Spacers, subtotals, and footnotes come through as rows too. So the row filter is not tidying-up, it is the thing that ends the read: filter to what a data row looks like (`WHERE \"Order ID\" LIKE 'SO-%'`) rather than to `IS NOT NULL`, which keeps any footnote carrying text in the first column. A bound that falls SHORT of the data is the dangerous direction: the rows and columns past it are dropped with no error at all, so overshoot the row bound and let the filter end the read.\n- Every number in an xlsx is stored as a double, so there are no integer columns. Typing is per column and decided by the FIRST data row, and `$1,234`, `12%` and `N/A` are all text: a text cell in that first row makes the whole column a string (on one real export, all ten of them), while a text cell further down leaves the column numeric and makes the read throw instead (`Could not convert string ... to DOUBLE`). `ignore_errors = true` fixes that second case, nulling the bad cells and keeping the column a number. It does nothing for the first.\n- Sample the column's SHAPES before writing any conversion, not its values: `run: source -> { group_by: shape is replace(raw_col, r'[0-9]', '9'); aggregate: n is count(); order_by: n desc }` collapses every value to its format and counts it, so on one real price column the 16 euro-denominated rows surface beside the 1,484 in dollars. A plain `group_by raw_col; limit: 20` sorts lexicographically, which hides exactly the shapes that matter.\n- Convert in the SQL source, not in Malloy, where `::number` throws on the first bad cell. `try_cast(regexp_replace(\"Total Revenue\", '[^0-9.-]', '', 'g') AS double)` nulls what it cannot read instead of failing and is right for a plain `$1,234.56`, but it is not a general parser. It concatenates every digit in the cell, so `1,234 (see tab 2)` becomes 12342. It understands only a leading ASCII `-`, so an accounting `(1,234)`, a Unicode minus and a `CR` suffix all come back positive, while a trailing `-` (`1,234-`) comes back null and drops the row from the sum. And it assumes `.` is the decimal point, so a European `1.234,56` comes back a thousandfold small. Handle the shapes your sample actually found, and divide a percent by 100. Failure is quiet either way: a cast that fails on every row sums to 0 rather than erroring, and a text date strips to a number rather than a null (`'01/02/2023'` becomes 1022023).\n- Check the answer against the sheet's own total row, read as raw text. Lift the data-row filter and select the footer by its label, which usually sits in a different column from the one your data-row predicate uses: on one export `WHERE \"Customer Name\" = 'TOTAL'` finds it and `WHERE \"Order ID\" = 'TOTAL'` returns nothing, and an empty result reads as a pass. Do not run the total through the same expression, because a wrong sign survives a row count, survives `select: *`, and cancels out when both sides are parsed the same broken way.\n- A sheet with no header row whose first row is all text silently loses that row to header detection. Pass `header = false`.\n- Headers with spaces are kept verbatim: backtick them in Malloy, or pass `normalize_names = true` for snake_case names.\n- `all_varchar = true` hands back each cell's stored value as text, so a date arrives as its raw Excel serial number rather than a date: `'44929'` from a sheet Excel wrote, `'44927.0'` from one DuckDB's own xlsx writer wrote, and `'44929.5'` where the cell carries a time of day. Which form you get depends on the tool that wrote the file, so do not detect serials by matching for an integer; `try_cast(... AS double)` accepts all three and returns null for a cell that was stored as text (`'01/02/2023'`), which is the test you want. Convert with `date '1899-12-30' + floor(try_cast(d AS double))::int`, not from 1900-01-01. Both wrappers earn their place: adding a double to a date does not compile, and a bare `::int` rounds, so an afternoon timestamp would land on the next day.\n- A date column that mixes both, which is what an export edited by hand gives you, needs both branches or you silently lose every row of one kind: `CASE WHEN try_cast(d AS double) IS NOT NULL THEN date '1899-12-30' + floor(try_cast(d AS double))::int ELSE try_strptime(d, '%m/%d/%Y')::date END`. Without `all_varchar`, a uniformly date-formatted column arrives as real `date` and `timestamp` values, and a stray text cell behaves exactly as the typing rule above says. Note what `ignore_errors = true` does here: it nulls that cell rather than parsing it, so the hand-typed date is lost silently.\n\n## Duplicate Rows: Check Before Building Measures\n\n```malloy\nrun: source -> { group_by: pk_field, aggregate: n is count(), having: n > 1, limit: 10 }\n```\n\nSymptoms: `sum()` returns astronomical values. Causes: event tables, batch retries, merged sources.\n\n## Mixed-Grain Joins: A Pre-Aggregated Source Ignores Your Filters\n\nJoining an aggregate-grain source (a decade/month/region summary table) into a detail-grain source produces values that do **not** respond to the query's filters. Malloy's symmetric aggregates prevent fan-out; they cannot prevent this, because the joined value is unfiltered *by construction*: it was computed over the whole population before the query ran.\n\n```\nrun: track_analysis -> {\n  where: genre = 'Rock'\n  group_by: decade\n  aggregate: track_count                    // filtered: Rock only     -> 701\n  group_by: decade_trends.decade_track_count // unfiltered population  -> 1,088\n}\n```\n\nTwo count-shaped numbers side by side, one filtered and one not; read as \"701 of 1,088 Rock tracks\" it is simply wrong: 1,088 is every genre. Two legitimate resolutions:\n\n- **Keep the join as a population baseline** when comparing a row to the whole population is the intent (e.g. `energy_vs_decade`). Then every joined field's `#(doc)` must say it is a fixed population value that does not respond to filters, and count-shaped fields with no comparison purpose (like `decade_track_count`) should be `internal:`; they only invite the misreading.\n- **Compute the aggregate as a query-based source from the detail table** so it derives from one source of truth and the derivation is visible.\n\nThis is the modeling-time consequence of ignoring `skill:malloy-scope`'s advice to skip pre-aggregated snapshot tables and compute fresh in Malloy instead.\n\n## Thresholds Are Decisions, Not Syntax\n\nBefore writing a `pick` expression or filtered measure with a numeric cutoff, see `skill:malloy-model` § Key Rules: every boundary must be user-supplied, distribution-derived (query the percentiles first), or explicitly flagged as an assumption in its `#(doc)`. Never invent one silently.\n\n## `except:` Removes Fields From Namespace Entirely\n\n`except:` in `include {}` completely removes fields: dimensions and measures cannot reference excluded fields. Use `internal:` instead when derived dimensions need the raw column.\n\n```malloy\n// WRONG: dimension references excluded field\nsource: x is conn.table('t')\ninclude { except: raw_date }\nextend { dimension: order_date is raw_date::date }  // ERROR! raw_date is gone\n\n// RIGHT: internal fields are still available in extend\nsource: x is conn.table('t')\ninclude { internal: raw_date }\nextend { dimension: order_date is raw_date::date }  // Works\n```\n\n## Source Order: Define Joined Tables First\n\nMalloy compiles top-to-bottom. Define lookup/dimension tables before the source that joins them, or use `import` statements in multi-file projects.\n\n## MUST Search Docs Before Using Unfamiliar Patterns\n\nCall `search_malloy_docs` BEFORE first use of any of these. Don't guess the syntax:\n- `pick` expressions\n- Window functions (`calculate`)\n- `percentile` or statistical functions: but see the hard limit above, raw-SQL aggregates (`sql_number` / `is_aggregate` / `percentile_cont!`) do **not** compile as measures in this build; there is no scalar median (`stddev` is the exception and does work as a measure)\n- Time interval functions (`days()`, `months()`): always `unit(start to end)`, and calendar units need date operands (see above)\n- Query-based sources (`source: x is (q -> {...}) extend {...}`; `from()` was removed and no longer parses)\n- `!` operator / `sql_number()`"
     },
     {
       name: "malloy-gotchas-queries",
@@ -296571,7 +296632,7 @@ Available: _concepts, build-derived-tables, build-unnest, curate-visibility, dis
     {
       name: "malloy-lookml-review/_concepts",
       description: "LookML → Malloy Concept Mapping. Reference detail for the malloy-lookml-review skill.",
-      body: "# LookML → Malloy Concept Mapping\n\nReference table for translating LookML constructs to Malloy. Referenced by multiple reference files.\n\n| LookML | Malloy | Notes |\n|--------|--------|-------|\n| `view:` | `source:` (base source file) | One source per physical table |\n| `explore:` | `source:` (source file with joins) | One source per analytical domain |\n| `dimension:` | `dimension:` | Direct mapping |\n| `dimension_group: { type: time }` | `.month`, `.year`, `::date` (native) | Malloy handles time natively; no explicit timeframe list needed |\n| `dimension: { type: yesno }` | `dimension: x is condition` | Boolean expression |\n| `measure: { type: count }` | `count()` | Always distinct in Malloy |\n| `measure: { type: count_distinct }` | `count(field)` | Direct mapping |\n| `measure: { type: sum }` | `sum(field)` | Direct mapping |\n| `measure: { type: average }` | `avg(field)` | Direct mapping |\n| `measure: { type: number }` | Derived measure expression | Usually a ratio; use `nullif()` for division |\n| `measure: { filters: [...] }` | `measure { where: condition }` | Filtered aggregate |\n| `primary_key: yes` | `primary_key: field_name` | Direct mapping |\n| `hidden: yes` | `# hidden` tag (cosmetic) | Classify reason first; see `curate-visibility.md` |\n| `fields` exclusion (explore/join) | `internal:` (with access modifiers) | Structurally excluded; `internal:` candidate |\n| `required_access_grants` | `private:` (with access modifiers) | Security-restricted; `private:` candidate |\n| `description:` | `#(doc)` tag | Direct mapping |\n| `label:` (simple rename) | `internal:` old + `dimension: new_name is old_name` | Lighter than `rename:`, and keeps the raw column reachable |\n| `label:` (complex) | `# label=\"Display Name\"` | When name differs from identifier |\n| `sql_table_name:` | `conn.table('schema.table')` | Use the connection name from the model definition if available |\n| `join: { relationship: many_to_one }` | `join_one:` | Direct mapping |\n| `join: { relationship: one_to_one }` | `join_one:` | Direct mapping |\n| `join: { relationship: one_to_many }` | `join_many:` | Direct mapping |\n| `join: { relationship: many_to_many }` | `join_cross:` | Direct mapping |\n| `sql_on: ${a.field} = ${b.field}` | `on a_field = b.b_field` | Translate `${}` references |\n| `CASE WHEN ... END` (in SQL) | `pick ... when ... else` | Direct syntax translation |\n| `COALESCE(a, b)` | `a ?? b` | Direct mapping |\n| `IFNULL(a, b)` | `a ?? b` | Direct mapping |\n| `${TABLE}.field` | `field` (direct column reference) | Malloy references columns directly |\n| `${view_name.field}` | `view_name.field` (join path) | In join conditions and cross-source refs |\n| `+view:` (refinement) | User decides: consolidate or `extend` | Malloy `extend` serves the same purpose |\n| `derived_table: { sql: ... }` (perf-only) | Use base table directly | PDT optimization is Looker-specific |\n| `derived_table: { sql: ... }` (transformation) | Flag for user | Recommend base table + dims or upstream dbt |\n| `derived_table: { explore_source: ... }` (NDT) | `from(source -> { group_by:, aggregate: }) extend { }` | Computed source pattern |\n| `value_format: \"$#,##0.00\"` | `# currency` | Map to Malloy render tags |\n| `value_format: \"0.00%\"` | `# percent` | Map to Malloy render tags |\n| `value_format_name: decimal_2` | `# number=\"0.00\"` | Map to Malloy render tags |"
+      body: "# LookML → Malloy Concept Mapping\n\nReference table for translating LookML constructs to Malloy. Referenced by multiple reference files.\n\n| LookML | Malloy | Notes |\n|--------|--------|-------|\n| `view:` | `source:` (base source file) | One source per physical table |\n| `explore:` | `source:` (source file with joins) | One source per analytical domain |\n| `dimension:` | `dimension:` | Direct mapping |\n| `dimension_group: { type: time }` | `.month`, `.year`, `::date` (native) | Malloy handles time natively; no explicit timeframe list needed |\n| `dimension: { type: yesno }` | `dimension: x is condition` | Boolean expression |\n| `measure: { type: count }` | `count()` | Always distinct in Malloy |\n| `measure: { type: count_distinct }` | `count(field)` | Direct mapping |\n| `measure: { type: sum }` | `sum(field)` | Direct mapping |\n| `measure: { type: average }` | `avg(field)` | Direct mapping |\n| `measure: { type: number }` | Derived measure expression | Usually a ratio; use `nullif()` for division |\n| `measure: { filters: [...] }` | `measure { where: condition }` | Filtered aggregate |\n| `primary_key: yes` | `primary_key: field_name` | Direct mapping |\n| `hidden: yes` | `# hidden` tag (cosmetic) | Classify reason first; see `curate-visibility.md` |\n| `fields` exclusion (explore/join) | `internal:` (with access modifiers) | Structurally excluded; `internal:` candidate |\n| `required_access_grants` | `private:` (with access modifiers) | Security-restricted; `private:` candidate |\n| `description:` | `#(doc)` tag | Direct mapping |\n| `label:` (simple rename) | `internal:` old + `dimension: new_name is old_name` | Lighter than `rename:`, and keeps the raw column reachable |\n| `label:` (complex) | `# label=\"Display Name\"` | When name differs from identifier |\n| `sql_table_name:` | `conn.table('schema.table')` | Use the connection name from the model definition if available |\n| `join: { relationship: many_to_one }` | `join_one:` | Direct mapping |\n| `join: { relationship: one_to_one }` | `join_one:` | Direct mapping |\n| `join: { relationship: one_to_many }` | `join_many:` | Direct mapping |\n| `join: { relationship: many_to_many }` | `join_cross:` | Direct mapping |\n| `sql_on: ${a.field} = ${b.field}` | `on a_field = b.b_field` | Translate `${}` references |\n| `CASE WHEN ... END` (in SQL) | `pick ... when ... else` | Direct syntax translation |\n| `COALESCE(a, b)` | `a ?? b` | Direct mapping |\n| `IFNULL(a, b)` | `a ?? b` | Direct mapping |\n| `${TABLE}.field` | `field` (direct column reference) | Malloy references columns directly |\n| `${view_name.field}` | `view_name.field` (join path) | In join conditions and cross-source refs |\n| `+view:` (refinement) | User decides: consolidate or `extend` | Malloy `extend` serves the same purpose |\n| `derived_table: { sql: ... }` (perf-only) | Use base table directly | PDT optimization is Looker-specific |\n| `derived_table: { sql: ... }` (transformation) | Flag for user | Recommend base table + dims or upstream dbt |\n| `derived_table: { explore_source: ... }` (NDT) | `(source -> { group_by:, aggregate: }) extend { }` | Computed source pattern |\n| `value_format: \"$#,##0.00\"` | `# currency` | Map to Malloy render tags |\n| `value_format: \"0.00%\"` | `# percent` | Map to Malloy render tags |\n| `value_format_name: decimal_2` | `# number=\"0.00\"` | Map to Malloy render tags |"
     },
     {
       name: "malloy-lookml-review/build-derived-tables",
@@ -296585,7 +296646,7 @@ Available: _concepts, build-derived-tables, build-unnest, curate-visibility, dis
 \`\`\`
 derived_table:
 ├── explore_source: → NDT path
-│   ├── Simple aggregation → Malloy from() extend {}
+│   ├── Simple aggregation → Malloy (query) extend {}
 │   ├── With derived_column: (window functions) → Malloy window function patterns
 │   ├── Chained NDTs → dependency ordering, multi-stage computed source
 │   └── With bind_filters → flag, no direct Malloy equivalent
@@ -296598,10 +296659,12 @@ derived_table:
 
 ### Simple Aggregation NDT
 
-Express the aggregation as a Malloy query, then build a source from it:
+Express the aggregation as a Malloy query, then build a source from it by
+wrapping the query in parentheses and extending it (\`from(...)\` was removed from
+the language and no longer parses):
 
 \`\`\`malloy
-source: source_name is from(
+source: source_name is (
   base_source -> {
     group_by: group_field
     aggregate:
@@ -296969,7 +297032,7 @@ Present coverage in this order:
     {
       name: "malloy-materialization",
       description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.",
-      body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n   - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n   - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"materialization\": {\n       \"scope\": \"package\",  // default; \"version\" = each published version owns its own tables\n       \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n       \"queryMetadata\": { \"team\": \"finance\" }  // tags the build's backend statements\n     }\n   }\n   ```\n   Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n   - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n   - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n   - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n   - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting an `#(authorize)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n  source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n  destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n  stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n  wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n  in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n  **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n  the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n  through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n  itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n  re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n  and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Two details\n  decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n  keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n  so a package-level `stale_ok` silently defeats a window you set on the source. And prefer the\n  **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n  the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n  it package-wide forces every other persisted source to recompute once stale too.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n  decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n  window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n  `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n  re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n  rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **An `#(authorize)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting an `#(authorize)`-gated source**."
+      body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n   - `##! experimental.persistence`, or\n   - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n   **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n   ```malloy\n   #@ persist name=\"my_dataset.my_table\"\n   source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n   ```\n   - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n   - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n   - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n   ```jsonc\n   {\n     \"name\": \"my-package\",\n     \"materialization\": {\n       \"scope\": \"package\",  // default; \"version\" = each published version owns its own tables\n       \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n       \"queryMetadata\": { \"team\": \"finance\" }  // tags the build's backend statements\n     }\n   }\n   ```\n   Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n   - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n   - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n   - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n   - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting an `#(authorize)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n  source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n  destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n  stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n  wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n  in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n  **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n  the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n  through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n  itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n  re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n  and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n  decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n  keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n  so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n  **layers**, which do not combine - not about siblings, below. Prefer the\n  **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n  the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n  it package-wide forces every other persisted source to recompute once stale too. And **a\n  content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n  content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n  two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n  policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n  cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n  `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n  layer rule above does not carry over here. If two sources need genuinely different windows, give them\n  genuinely different SQL.\n\n  Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n  the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n  manifest-assembly time typically applies it when a version's manifest is next published rather than\n  retroactively to manifests already distributed - so you can declare the window correctly and not have it\n  in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n  decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n  window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n  `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n  re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n  rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **An `#(authorize)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting an `#(authorize)`-gated source**."
     },
     {
       name: "malloy-materialization-tuning",
@@ -297083,10 +297146,12 @@ extend {
 
 ### Computed Source (from Query)
 
+Wrap the query in parentheses and extend it. \`from(...)\` was removed from the language and no longer parses (\`unexpected 'from'\`).
+
 \`\`\`malloy
 import "orders.malloy"
 
-source: user_order_facts is from(
+source: user_order_facts is (
   orders -> {
     group_by: customer_id
     aggregate:
@@ -297943,7 +298008,7 @@ Steps to follow when the user asks a question:
   - If the question can be answered by a query already in the notebook, run that cell's query via \`execute_query\` (exact code, or a minor variation like adding a filter or changing a group_by).
   - If the question asks for an analysis that is clearly NOT in the notebook (new source, different package, different domain), then, and only then, call \`get_context\` to explore.
   - Do NOT call \`get_context\` as a default first step. The notebook already tells you what's available.
-4. Before writing or modifying a query, read the \`malloy-queries\` skill for syntax patterns. When you tweak a query (add a \`where:\` clause, change a \`group_by\`, etc.), do NOT add \`#(filter)\` annotations: filters live on the source's model file and are inherited by this notebook automatically. Query-level \`where:\` filtering inside a cell is fine; declaring new filter UI is a model change, not a chat-time change.
+4. Before writing or modifying a query, read the \`malloy-queries\` skill for syntax patterns. When you tweak a query (add a \`where:\` clause, change a \`group_by\`, etc.), do NOT add \`#(filter)\` annotations or \`given:\` declarations: both live on the source's model file and are inherited by this notebook automatically. Query-level \`where:\` filtering inside a cell is fine; declaring a new runtime parameter is a model change, not a chat-time change.
 5. Summarize insights from query results. Do not echo raw rows: the user sees them rendered.`
     },
     {
@@ -298580,7 +298645,7 @@ For every rule, the linked instruction-skill section is the canonical source for
 - **Why this isn't a correctness rule.** The Malloy compiler auto-synthesizes a UUID-based \`__distinct_key\` for any \`join_one:\` target without a declared PK (\`packages/malloy/src/model/field_instance.ts:644-670\`, \`query_query.ts:946-951\`), so symmetric aggregation is correct either way, see the C-07 entry in \`rubric-correctness.md\`'s "Rules we dropped" section. The \`with\` shortcut, which DOES require a declared PK, errors at compile time and surfaces as a diagnostic. The actual silent-correctness hazard ("declared PK isn't actually unique in the data") is \`rubric-correctness.md\` § C-12.
 - **What this rule catches.** A discoverability / hygiene gap: when a source has a natural primary key, declaring it lets downstream code use the \`with\` shortcut, makes grain explicit in the model, and gives tooling a stable identifier per row. Treat as a recommendation, not a merge gate.
 - **Detection:** for every \`source:\` declaration, check whether its body contains a \`primary_key:\` clause. Skip flagging when:
-  - The source is a query-based / computed source (\`source: x is t -> {...}\` or \`source: x is from(...)\`) where grain is determined by the \`group_by\` columns, declaring a \`primary_key:\` on the result is fine but not required.
+  - The source is a query-based / computed source (\`source: x is t -> {...}\` or \`source: x is (t -> {...}) extend {...}\`) where grain is determined by the \`group_by\` columns, declaring a \`primary_key:\` on the result is fine but not required.
   - The source represents an event/log table or a denormalized analytical source where no natural PK exists. Both situations are legitimate; the LLM should recognize them and skip the finding.
 - **Fix (when the source does have a natural PK):** declare it, \`primary_key: <col>\` inside \`extend {}\`. When there isn't a natural PK, leave it undeclared; if the absence is non-obvious, a one-line \`#(doc)\` on the source explaining the grain helps future readers.
 - **See:** \`skill:malloy-model\` § Key Rules · \`rubric-correctness.md\` § C-12 (the related correctness rule that checks whether a declared PK is actually unique in the data) · \`rubric-style.md\` § Y-03 (\`join_one:\` style consistency, which is the other consequence of declared-vs-undeclared PKs)
@@ -299274,6 +299339,39 @@ function parseNonNegativeIntParam(value) {
   const parsed = parseInt(String(value), 10);
   return Number.isInteger(parsed) && parsed >= 0 ? parsed : undefined;
 }
+function parseBooleanParam(value) {
+  if (value === undefined || value === null)
+    return { ok: true, value: false };
+  if (value === "true")
+    return { ok: true, value: true };
+  if (value === "false")
+    return { ok: true, value: false };
+  return { ok: false };
+}
+function invalidBooleanMessage(name, value, method, routePath) {
+  return `Invalid ${name} value ${JSON.stringify(value)}: expected "true" or ` + `"false". Fix: ${method} ${routePath}?${name}=true.`;
+}
+
+// src/route_params.ts
+init_errors();
+function optionalBooleanParamOr400(req, res, name) {
+  const raw = req.query[name];
+  const parsed = parseBooleanParam(raw);
+  if (parsed.ok) {
+    return { ok: true, value: raw === undefined ? undefined : parsed.value };
+  }
+  const { json, status } = internalErrorToHttpError(new BadRequestError(invalidBooleanMessage(name, raw, req.method, req.path)));
+  res.status(status).json(json);
+  return { ok: false };
+}
+function booleanParamOr400(req, res, name) {
+  const outcome = optionalBooleanParamOr400(req, res, name);
+  return outcome.ok ? outcome.value ?? false : undefined;
+}
+function setCollectionReloadError(res, perResourceRoute) {
+  const { json, status } = internalErrorToHttpError(new BadRequestError(`Reload recompiles one named resource, and this endpoint lists them. ` + `Use GET ${perResourceRoute}?reload=true instead.`));
+  res.status(status).json(json);
+}
 
 // src/server-old.ts
 init_connection_config();
@@ -299306,7 +299404,11 @@ function registerLegacyRoutes(app, controllers) {
     compileController,
     materializationController
   } = controllers;
-  app.get(`${LEGACY_API_PREFIX}/projects`, async (_req, res) => {
+  app.get(`${LEGACY_API_PREFIX}/projects`, async (req, res) => {
+    if (req.query.reload !== undefined) {
+      setCollectionReloadError(res, `${LEGACY_API_PREFIX}/projects/{projectName}`);
+      return;
+    }
     try {
       res.status(200).json(await environmentStore.listEnvironments());
     } catch (error) {
@@ -299328,8 +299430,12 @@ function registerLegacyRoutes(app, controllers) {
     }
   });
   app.get(`${LEGACY_API_PREFIX}/projects/:projectName`, async (req, res) => {
+    const reload = booleanParamOr400(req, res, "reload");
+    if (reload === undefined) {
+      return;
+    }
     try {
-      const environment = await environmentStore.getEnvironment(req.params.projectName, req.query.reload === "true");
+      const environment = await environmentStore.getEnvironment(req.params.projectName, reload);
       res.status(200).json(await environment.serialize());
     } catch (error) {
       logger.error(error);
@@ -299550,6 +299656,10 @@ function registerLegacyRoutes(app, controllers) {
       setVersionIdError(res);
       return;
     }
+    if (req.query.reload !== undefined) {
+      setCollectionReloadError(res, `${LEGACY_API_PREFIX}/projects/${req.params.projectName}/packages/{packageName}`);
+      return;
+    }
     try {
       res.status(200).json(await packageController.listPackages(req.params.projectName));
     } catch (error) {
@@ -299573,8 +299683,12 @@ function registerLegacyRoutes(app, controllers) {
       setVersionIdError(res);
       return;
     }
+    const reload = booleanParamOr400(req, res, "reload");
+    if (reload === undefined) {
+      return;
+    }
     try {
-      res.status(200).json(await packageController.getPackage(req.params.projectName, req.params.packageName, req.query.reload === "true"));
+      res.status(200).json(await packageController.getPackage(req.params.projectName, req.params.packageName, reload));
     } catch (error) {
       logger.error(error);
       const { json, status } = internalErrorToHttpError(error);
@@ -299686,7 +299800,11 @@ function registerLegacyRoutes(app, controllers) {
           return;
         }
       }
-      const bypassFilters = req.query.bypass_filters === "true" ? true : undefined;
+      const bypass = optionalBooleanParamOr400(req, res, "bypass_filters");
+      if (!bypass.ok) {
+        return;
+      }
+      const bypassFilters = bypass.value;
       res.status(200).json(await modelController.executeNotebookCell(req.params.projectName, req.params.packageName, notebookPath, cellIndex, filterParams, bypassFilters));
     } catch (error) {
       logger.error(error);
@@ -299765,8 +299883,12 @@ function registerLegacyRoutes(app, controllers) {
     }
   });
   app.delete(`${LEGACY_API_PREFIX}/projects/:projectName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
+    const dropTables = booleanParamOr400(req, res, "dropTables");
+    if (dropTables === undefined) {
+      return;
+    }
     try {
-      await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
+      await materializationController.deleteMaterialization(req.params.projectName, req.params.packageName, req.params.materializationId, { dropTables });
       res.status(204).send();
     } catch (error) {
       const { json, status } = internalErrorToHttpError(error);
@@ -300939,6 +301061,13 @@ function reportRetainedStorageTables(retained, packageName) {
 function isReclaimableStorageTable(entry) {
   return !!entry.storageDestinationName && entry.refresh === undefined;
 }
+function physicalTargetKey(instruction, connectionName) {
+  return JSON.stringify(instruction.destination ? [
+    "destination",
+    instruction.destination,
+    instruction.physicalTableName
+  ] : ["connection", connectionName, instruction.physicalTableName]);
+}
 
 class MaterializationService {
   environmentStore;
@@ -301386,6 +301515,17 @@ class MaterializationService {
       if (instruction.sourceID) {
         bySourceID.set(instruction.sourceID, instruction);
       }
+      const clash = bySourceEntityId.get(instruction.sourceEntityId);
+      if (clash && clash.physicalTableName !== instruction.physicalTableName) {
+        recordSharedAddressInstructions();
+        logger.warn("One content address was instructed to build more than one table", {
+          sourceEntityId: instruction.sourceEntityId,
+          physicalTableNames: [
+            clash.physicalTableName,
+            instruction.physicalTableName
+          ]
+        });
+      }
       bySourceEntityId.set(instruction.sourceEntityId, instruction);
     }
     const manifest = new Manifest;
@@ -301404,6 +301544,49 @@ class MaterializationService {
     const failures = {};
     const failedReasons = [];
     const builtSources = [];
+    const writtenTargets = new Map;
+    const claimedBy = new Map;
+    const collisions = [];
+    for (const graph of graphs) {
+      for (const persistSource of iterGraphSources(graph, sources)) {
+        let address;
+        try {
+          address = computeSourceEntityId(persistSource, connectionDigests);
+        } catch {
+          continue;
+        }
+        const instruction = bySourceID.get(persistSource.sourceID) ?? bySourceEntityId.get(address);
+        if (!instruction)
+          continue;
+        const target = physicalTargetKey(instruction, graph.connectionName);
+        const claim = claimedBy.get(target);
+        if (!claim) {
+          claimedBy.set(target, {
+            sourceName: persistSource.name,
+            sourceEntityId: address
+          });
+        } else if (claim.sourceEntityId !== address) {
+          collisions.push({
+            first: claim.sourceName,
+            second: persistSource.name,
+            table: instruction.physicalTableName
+          });
+        }
+      }
+    }
+    for (const c of collisions) {
+      recordTableCollision();
+      logger.warn("Two definitions are materializing into one table", {
+        physicalTableName: c.table,
+        sourceNames: [c.first, c.second]
+      });
+    }
+    if (collisions.length > 0 && getPersistCollisionEnforce()) {
+      const detail = collisions.map((c) => `'${c.first}' and '${c.second}' compile to different SQL but ` + `both materialize into table '${c.table}'`).join("; ");
+      throw new MaterializationEligibilityError({
+        message: `${detail}, so each would overwrite the other's rows while both ` + `resolve to it at serve time. Give them distinct definitions, or ` + `distinct physical names: a model-declared collision is fixed ` + `with '#@ persist name=', a host-assigned one by the caller that ` + `minted the names.`
+      });
+    }
     try {
       for (const graph of graphs) {
         const connection = connections.get(graph.connectionName);
@@ -301438,6 +301621,17 @@ class MaterializationService {
           if (!orchestratedInstruction && instruction.destination && getPersistStorageMode() !== "off") {
             assertMaterializationEligible(persistSource);
           }
+          const target = physicalTargetKey(instruction, graph.connectionName);
+          const written = writtenTargets.get(target);
+          if (written && written.sourceEntityId === sourceEntityId) {
+            recordDuplicateTargetSkipped();
+            logger.debug("Skipping a source whose table this run built", {
+              sourceName: persistSource.name,
+              builtAs: written.sourceName,
+              physicalTableName: instruction.physicalTableName
+            });
+            continue;
+          }
           let entry;
           try {
             entry = await this.buildOneSource(persistSource, instruction, connection, connectionDigests, manifest, environment, entries, buildMetadata, incremental, sourceEntityId);
@@ -301475,6 +301669,10 @@ class MaterializationService {
           }
           builtSources.push(persistSource.name);
           entries[sourceEntityId] = entry;
+          writtenTargets.set(target, {
+            sourceEntityId,
+            sourceName: persistSource.name
+          });
           if (isReclaimableStorageTable(entry)) {
             builtThisRun.push(entry);
           } else if (entry.storageDestinationName) {
@@ -301957,7 +302155,7 @@ class MaterializationService {
     };
   }
   async buildDownstreamViaParents(persistSource, destinationName, destinationConnection, builtEntries, environment, physicalTableName) {
-    const upstreams = deriveServeBindings(builtEntries).filter((b) => b.destinationName === destinationName);
+    const upstreams = deriveServeBindings(builtEntries, {}).filter((b) => b.destinationName === destinationName);
     if (upstreams.length === 0) {
       throw new MaterializationEligibilityError({
         message: "no materialized upstream is available in this destination to build on"
@@ -303731,7 +303929,11 @@ data: changed
   };
   req.on("close", cleanup);
 });
-app.get(`${API_PREFIX2}/environments`, async (_req, res) => {
+app.get(`${API_PREFIX2}/environments`, async (req, res) => {
+  if (req.query.reload !== undefined) {
+    setCollectionReloadError(res, `${API_PREFIX2}/environments/{environmentName}`);
+    return;
+  }
   try {
     res.status(200).json(await environmentStore.listEnvironments());
   } catch (error) {
@@ -303753,8 +303955,12 @@ app.post(`${API_PREFIX2}/environments`, async (req, res) => {
   }
 });
 app.get(`${API_PREFIX2}/environments/:environmentName`, async (req, res) => {
+  const reload = booleanParamOr400(req, res, "reload");
+  if (reload === undefined) {
+    return;
+  }
   try {
-    const environment = await environmentStore.getEnvironment(req.params.environmentName, req.query.reload === "true");
+    const environment = await environmentStore.getEnvironment(req.params.environmentName, reload);
     res.status(200).json(await environment.serialize());
   } catch (error) {
     logger.error(error);
@@ -303963,6 +304169,10 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages`, async (req, res
     setVersionIdError2(res);
     return;
   }
+  if (req.query.reload !== undefined) {
+    setCollectionReloadError(res, `${API_PREFIX2}/environments/${req.params.environmentName}/packages/{packageName}`);
+    return;
+  }
   try {
     res.status(200).json(await packageController.listPackages(req.params.environmentName));
   } catch (error) {
@@ -303997,8 +304207,12 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName`, as
     setVersionIdError2(res);
     return;
   }
+  const reload = booleanParamOr400(req, res, "reload");
+  if (reload === undefined) {
+    return;
+  }
   try {
-    res.status(200).json(await packageController.getPackage(req.params.environmentName, req.params.packageName, req.query.reload === "true"));
+    res.status(200).json(await packageController.getPackage(req.params.environmentName, req.params.packageName, reload));
   } catch (error) {
     logger.error(error);
     const { json, status } = internalErrorToHttpError(error);
@@ -304114,7 +304328,11 @@ app.get(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/note
         return;
       }
     }
-    const bypassFilters = req.query.bypass_filters === "true" ? true : undefined;
+    const bypass = optionalBooleanParamOr400(req, res, "bypass_filters");
+    if (!bypass.ok) {
+      return;
+    }
+    const bypassFilters = bypass.value;
     let givens;
     if (typeof req.query.givens === "string") {
       try {
@@ -304242,8 +304460,12 @@ app.post(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/mat
   }
 });
 app.delete(`${API_PREFIX2}/environments/:environmentName/packages/:packageName/materializations/:materializationId`, async (req, res) => {
+  const dropTables = booleanParamOr400(req, res, "dropTables");
+  if (dropTables === undefined) {
+    return;
+  }
   try {
-    await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables: req.query.dropTables === "true" });
+    await materializationController.deleteMaterialization(req.params.environmentName, req.params.packageName, req.params.materializationId, { dropTables });
     res.status(204).send();
   } catch (error) {
     const { json, status } = internalErrorToHttpError(error);