@malloy-publisher/server 0.0.223 → 0.0.225

package/dist/server.mjs

@@ -158407,6 +158407,11 @@ var require_utils74 = __commonJS((exports, module) => {
   };
 });
 
+// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
+var require_sshcrypto = __commonJS((exports, module) => {
+  module.exports = __require("./sshcrypto-8m50vnmb.node");
+});
+
 // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
 var require_poly1305 = __commonJS((exports, module) => {
   var __dirname = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto", __filename = "/home/runner/work/publisher/publisher/node_modules/ssh2/lib/protocol/crypto/poly1305.js";
@@ -158893,7 +158898,7 @@ var require_crypto = __commonJS((exports, module) => {
   var ChaChaPolyDecipher;
   var GenericDecipher;
   try {
-    binding = (()=>{throw new Error("Cannot require module "+"./crypto/build/Release/sshcrypto.node");})();
+    binding = require_sshcrypto();
     ({
       AESGCMCipher,
       ChaChaPolyCipher,
@@ -256953,7 +256958,12 @@ async function fetchManifestEntries(uri) {
       });
       continue;
     }
-    entries[sourceEntityId] = { tableName: physicalTableName };
+    entries[sourceEntityId] = {
+      tableName: physicalTableName,
+      dataAsOf: entry.dataAsOf,
+      freshnessWindowSeconds: entry.freshnessWindowSeconds,
+      freshnessFallback: entry.freshnessFallback
+    };
   }
   return entries;
 }
@@ -257537,6 +257547,7 @@ class Model {
   fileLevelAuthorize = [];
   discoveryCurationEnabled = false;
   queryBoundary = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
+  freshnessResolver;
   meter = publisherMeter();
   queryExecutionHistogram = this.meter.createHistogram("malloy_model_query_duration", {
     description: "How long it takes to execute a Malloy model query",
@@ -257749,6 +257760,13 @@ class Model {
   setDiscoveryCuration(enabled) {
     this.discoveryCurationEnabled = enabled;
   }
+  setFreshnessResolver(resolver) {
+    this.freshnessResolver = resolver;
+  }
+  resolveFreshBuildManifest() {
+    const entries = this.freshnessResolver?.();
+    return entries ? { entries, strict: false } : undefined;
+  }
   getSources() {
     return this.curateForDiscovery(this.sources);
   }
@@ -257993,12 +258011,18 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
     }
     const maxRows = getMaxQueryRows();
     const maxBytes = getMaxResponseBytes();
-    const rowLimit = resolveModelQueryRowLimit((await runnable.getPreparedResult({ givens })).resultExplore.limit, { defaultLimit: getDefaultQueryRowLimit(), maxRows });
+    const buildManifest = this.resolveFreshBuildManifest();
+    const rowLimit = resolveModelQueryRowLimit((await runnable.getPreparedResult({ givens, buildManifest })).resultExplore.limit, { defaultLimit: getDefaultQueryRowLimit(), maxRows });
     const endTime = performance.now();
     const executionTime = endTime - startTime;
     let queryResults;
     try {
-      queryResults = await runnable.run({ rowLimit, givens, abortSignal });
+      queryResults = await runnable.run({
+        rowLimit,
+        givens,
+        abortSignal,
+        buildManifest
+      });
     } catch (error) {
       const errorEndTime = performance.now();
       const errorExecutionTime = errorEndTime - startTime;
@@ -258125,14 +258149,19 @@ run: ${sourceName ? sourceName + "->" : ""}${queryName}`;
         }
         const cellMaxRows = getMaxQueryRows();
         const cellMaxBytes = getMaxResponseBytes();
-        const rowLimit = resolveModelQueryRowLimit((await runnableToExecute.getPreparedResult({ givens })).resultExplore.limit, {
+        const buildManifest = this.resolveFreshBuildManifest();
+        const rowLimit = resolveModelQueryRowLimit((await runnableToExecute.getPreparedResult({
+          givens,
+          buildManifest
+        })).resultExplore.limit, {
           defaultLimit: getDefaultQueryRowLimit(),
           maxRows: cellMaxRows
         });
         const result = await runnableToExecute.run({
           rowLimit,
           givens,
-          abortSignal
+          abortSignal,
+          buildManifest
         });
         const query = (await runnableToExecute.getPreparedQuery())._query;
         queryName = query.as || query.name;
@@ -258397,6 +258426,7 @@ function hydrateMarkdownOnlyCells(notebookCells) {
 }
 
 // src/service/build_plan.ts
+var FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"];
 function deriveColumns(persistSource) {
   try {
     return persistSource._explore.intrinsicFields.filter((f) => f.isAtomicField()).map((f) => ({
@@ -258423,6 +258453,65 @@ function deriveAnnotationFields(persistSource) {
   } catch {}
   return out;
 }
+function tagFreshnessScheduleLayer(tag) {
+  if (!tag || typeof tag.text !== "function")
+    return {};
+  const layer = {};
+  const window2 = tag.text("freshness", "window");
+  if (typeof window2 === "string")
+    layer.window = window2;
+  const fallback = tag.text("freshness", "fallback");
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  const schedule = tag.text("schedule");
+  if (typeof schedule === "string")
+    layer.schedule = schedule;
+  return layer;
+}
+function packageFreshnessLayer(cfg) {
+  if (!cfg)
+    return {};
+  const layer = {};
+  if (cfg.freshness?.window)
+    layer.window = cfg.freshness.window;
+  const fallback = cfg.freshness?.fallback;
+  if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
+    layer.fallback = fallback;
+  }
+  return layer;
+}
+function safeSourceTag(source) {
+  try {
+    return source.annotations.parseAsTag("@").tag;
+  } catch {
+    return;
+  }
+}
+function safeModelTag(source) {
+  try {
+    return source.modelAnnotations.parseAsTag().tag;
+  } catch {
+    return;
+  }
+}
+function resolveFreshnessSchedule(source, packageMaterialization) {
+  const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
+  const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
+  const pkgLayer = packageFreshnessLayer(packageMaterialization);
+  const window2 = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
+  const fallback = sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
+  const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
+  let freshness = null;
+  if (window2 !== undefined || fallback !== undefined) {
+    freshness = {};
+    if (window2 !== undefined)
+      freshness.window = window2;
+    if (fallback !== undefined)
+      freshness.fallback = fallback;
+  }
+  return { freshness, schedule };
+}
 function flattenDependsOn(node) {
   return node.dependsOn.map((d) => d.sourceID);
 }
@@ -258515,7 +258604,7 @@ async function compilePackageBuildPlan(pkg, signal) {
     sourceModelPaths
   };
 }
-function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths) {
+function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, sourceModelPaths, packageMaterialization) {
   const include = sourceNames ? new Set(sourceNames) : null;
   const wireGraphs = graphs.map((graph) => ({
     connectionName: graph.connectionName,
@@ -258529,6 +258618,7 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
     if (include && !include.has(source.name))
       continue;
     const annotationFields = deriveAnnotationFields(source);
+    const { freshness, schedule } = resolveFreshnessSchedule(source, packageMaterialization);
     wireSources[sourceID] = {
       name: source.name,
       sourceID: source.sourceID,
@@ -258538,6 +258628,8 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
       sql: source.getSQL(),
       sharing: annotationFields.sharing ?? null,
       refresh: annotationFields.refresh ?? null,
+      freshness,
+      schedule,
       columns: deriveColumns(source),
       annotationFields,
       modelPath: sourceModelPaths?.[sourceID]
@@ -258550,7 +258642,47 @@ async function computePackageBuildPlan(pkg, signal) {
   if (compiled.graphs.length === 0) {
     return null;
   }
-  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths);
+  return deriveBuildPlan(compiled.graphs, compiled.sources, compiled.connectionDigests, undefined, compiled.sourceModelPaths, pkg.getMaterializationConfig?.() ?? null);
+}
+
+// src/service/freshness.ts
+function evaluateManifestFreshness(entry, now) {
+  if (entry.freshnessWindowSeconds == null)
+    return "serve_table";
+  if (entry.dataAsOf == null)
+    return "serve_table";
+  const dataAsOfMs = Date.parse(entry.dataAsOf);
+  if (Number.isNaN(dataAsOfMs))
+    return "serve_table";
+  const ageSeconds = (now.getTime() - dataAsOfMs) / 1000;
+  const stale = ageSeconds > entry.freshnessWindowSeconds;
+  if (!stale)
+    return "serve_table";
+  return entry.freshnessFallback === "stale_ok" ? "serve_table" : "serve_live";
+}
+function staleSinceMs(entry) {
+  if (entry.freshnessWindowSeconds == null || entry.dataAsOf == null) {
+    return null;
+  }
+  const dataAsOfMs = Date.parse(entry.dataAsOf);
+  if (Number.isNaN(dataAsOfMs))
+    return null;
+  return dataAsOfMs + entry.freshnessWindowSeconds * 1000;
+}
+function filterFreshManifest(entries, now) {
+  const manifest = {};
+  let nextStaleSince = null;
+  const nowMs = now.getTime();
+  for (const [sourceEntityId, entry] of Object.entries(entries)) {
+    if (evaluateManifestFreshness(entry, now) === "serve_live")
+      continue;
+    manifest[sourceEntityId] = { tableName: entry.tableName };
+    const flipAt = staleSinceMs(entry);
+    if (flipAt != null && flipAt > nowMs) {
+      nextStaleSince = nextStaleSince == null ? flipAt : Math.min(nextStaleSince, flipAt);
+    }
+  }
+  return { manifest, nextStaleSince };
 }
 
 // src/service/persist_annotation_validation.ts
@@ -258575,6 +258707,14 @@ function assertPersistNamesQuoted(modelSource, modelPath) {
 }
 
 // src/service/package.ts
+function toTableNameManifest(entries) {
+  const out = {};
+  for (const [sourceEntityId, entry] of Object.entries(entries)) {
+    out[sourceEntityId] = { tableName: entry.tableName };
+  }
+  return out;
+}
+
 class Package {
   environmentName;
   packageName;
@@ -258584,6 +258724,8 @@ class Package {
   packagePath;
   malloyConfig;
   buildManifestEntries;
+  freshnessEntries;
+  freshManifestCache;
   manifestBindingStatus = "unbound";
   manifestEntryCount = 0;
   boundManifestUri = null;
@@ -258743,6 +258885,7 @@ class Package {
     });
     const pkg = new Package(environmentName, packageName, packagePath, packageConfig, databases, models, malloyConfig);
     pkg.renderTagWarnings = renderTagWarnings;
+    pkg.wireFreshnessResolvers();
     try {
       const buildPlanStart = Date.now();
       pkg.buildPlan = await computePackageBuildPlan(pkg);
@@ -258776,6 +258919,9 @@ class Package {
   getBuildPlan() {
     return this.buildPlan;
   }
+  getMaterializationConfig() {
+    return this.packageMetadata.materialization ?? null;
+  }
   getPackageMetadata() {
     const metadata = {
       ...this.packageMetadata,
@@ -258797,17 +258943,37 @@ class Package {
   getBuildManifestEntries() {
     return this.buildManifestEntries;
   }
+  getFreshBuildManifest(now = Date.now()) {
+    if (!this.freshnessEntries)
+      return;
+    if (this.freshManifestCache && now < this.freshManifestCache.validUntil) {
+      return this.freshManifestCache.manifest;
+    }
+    const { manifest, nextStaleSince } = filterFreshManifest(this.freshnessEntries, new Date(now));
+    this.freshManifestCache = {
+      manifest,
+      validUntil: nextStaleSince ?? Infinity
+    };
+    return manifest;
+  }
+  wireFreshnessResolvers() {
+    for (const model of this.models.values()) {
+      model.setFreshnessResolver(() => this.getFreshBuildManifest());
+    }
+  }
   setBoundManifestUri(uri) {
     this.boundManifestUri = uri;
   }
   markManifestBindFailed() {
     this.manifestBindingStatus = "live_fallback";
   }
-  recordManifestBinding(buildManifest) {
-    const count = Object.keys(buildManifest).length;
-    this.buildManifestEntries = count > 0 ? buildManifest : undefined;
+  recordManifestBinding(entries) {
+    const count = Object.keys(entries).length;
+    this.freshnessEntries = count > 0 ? entries : undefined;
+    this.buildManifestEntries = count > 0 ? toTableNameManifest(entries) : undefined;
     this.manifestEntryCount = count;
     this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
+    this.freshManifestCache = undefined;
     if (count === 0) {
       this.boundManifestUri = null;
     }
@@ -258841,11 +259007,22 @@ class Package {
 `);
   }
   scheduleWarnings() {
-    const schedule = this.packageMetadata.materialization?.schedule;
-    if (!schedule)
-      return [];
-    const sources = Object.values(this.buildPlan?.sources ?? {});
-    return sources.filter((source) => source.sharing !== "private").map((source) => `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` + `persist source to declare '#@ persist ... sharing=private'; source ` + `'${source.name}' resolves to ${source.sharing ? `'${source.sharing}'` : "unset"}. Declare 'materialization.freshness.window' instead (the control plane ` + `schedules refreshes from it), or mark every persist source sharing=private.`);
+    const warnings = [];
+    const sources = this.buildPlan?.sources ? Object.values(this.buildPlan.sources) : [];
+    for (const source of sources) {
+      if (source.schedule && source.sharing !== "private") {
+        warnings.push(`#@ persist source "${source.name}" declares a schedule (cron) but ` + `resolves to sharing="${source.sharing ?? "shared (unset default)"}": a per-source cron is valid only for sharing="private". ` + `Declare 'freshness.window' instead (the control plane schedules ` + `refreshes from it).`);
+      }
+    }
+    const packageSchedule = this.packageMetadata.materialization?.schedule;
+    if (packageSchedule) {
+      const nonPrivate = sources.filter((s) => s.sharing !== "private");
+      if (sources.length === 0 || nonPrivate.length > 0) {
+        const detail = sources.length === 0 ? "the package declares no persist source for it to govern" : `sources [${nonPrivate.map((s) => s.name).join(", ")}] resolve to shared/unset`;
+        warnings.push(`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` + `only when every persist source resolves to sharing="private": ` + `${detail}. Declare 'materialization.freshness.window' instead ` + `(the control plane schedules refreshes from it).`);
+      }
+    }
+    return warnings;
   }
   formatInvalidSchedule() {
     return this.scheduleWarnings().join(`
@@ -258891,7 +259068,8 @@ class Package {
   getModelPaths() {
     return Array.from(this.models.keys());
   }
-  async reloadAllModels(buildManifest) {
+  async reloadAllModels(entries) {
+    const buildManifest = toTableNameManifest(entries);
     const modelPaths = Array.from(this.models.keys());
     logger.info("Reloading all models with build manifest", {
       packageName: this.packageName,
@@ -258951,7 +259129,8 @@ class Package {
     this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
     this.applyDiscoveryPolicyToModels();
     this.applyQueryBoundaryToModels();
-    this.recordManifestBinding(buildManifest);
+    this.recordManifestBinding(entries);
+    this.wireFreshnessResolvers();
     const invalidMsg = this.formatInvalidExplores();
     if (invalidMsg) {
       logger.warn(`Package ${this.packageName} has invalid explores`, {
@@ -261163,7 +261342,14 @@ class MaterializationController {
   validateCreateBody(body) {
     const result = {};
     if (body.buildInstructions !== undefined && body.buildInstructions !== null) {
-      result.buildInstructions = this.validateBuildInstructions(body.buildInstructions);
+      const parsed = this.validateBuildInstructions(body.buildInstructions);
+      result.buildInstructions = parsed.sources;
+      if (parsed.referenceManifest !== undefined) {
+        result.referenceManifest = parsed.referenceManifest;
+      }
+      if (parsed.strictUpstreams !== undefined) {
+        result.strictUpstreams = parsed.strictUpstreams;
+      }
     }
     if (body.forceRefresh !== undefined) {
       if (typeof body.forceRefresh !== "boolean") {
@@ -261183,11 +261369,42 @@ class MaterializationController {
     if (typeof raw !== "object" || raw === null) {
       throw new BadRequestError("buildInstructions must be an object");
     }
-    const sources = raw.sources;
+    const obj = raw;
+    const sources = obj.sources;
     if (!Array.isArray(sources) || sources.length === 0) {
       throw new BadRequestError("buildInstructions requires a non-empty 'sources' array of BuildInstruction");
     }
-    return sources.map((instruction) => this.validateInstruction(instruction));
+    const result = {
+      sources: sources.map((instruction) => this.validateInstruction(instruction))
+    };
+    if (obj.referenceManifest !== undefined && obj.referenceManifest !== null) {
+      if (!Array.isArray(obj.referenceManifest)) {
+        throw new BadRequestError("buildInstructions.referenceManifest must be an array of ManifestReference");
+      }
+      result.referenceManifest = obj.referenceManifest.map((ref) => this.validateManifestReference(ref));
+    }
+    if (obj.strictUpstreams !== undefined) {
+      if (typeof obj.strictUpstreams !== "boolean") {
+        throw new BadRequestError("buildInstructions.strictUpstreams must be a boolean");
+      }
+      result.strictUpstreams = obj.strictUpstreams;
+    }
+    return result;
+  }
+  validateManifestReference(raw) {
+    if (typeof raw !== "object" || raw === null) {
+      throw new BadRequestError("Each manifest reference must be an object");
+    }
+    const ref = raw;
+    for (const field of ["sourceEntityId", "physicalTableName"]) {
+      if (typeof ref[field] !== "string") {
+        throw new BadRequestError(`Manifest reference is missing required string field '${field}'`);
+      }
+    }
+    return {
+      sourceEntityId: ref.sourceEntityId,
+      physicalTableName: ref.physicalTableName
+    };
   }
   validateInstruction(raw) {
     if (typeof raw !== "object" || raw === null) {
@@ -265777,7 +265994,162 @@ view: rev_delta is {
 }
 \`\`\`
 
-Use \`down_is_good=true\` for metrics where decrease is positive (churn, defects).` }, { name: "lookml-review", description: "Analyze LookML files as prior art for Malloy modeling. Used during Step 1 (DISCOVER) when .lkml files are present. Coordinates reference files that extract business logic, relationships, and curation decisions. Works with or without a database connection.", body: `# LookML Review
+Use \`down_is_good=true\` for metrics where decrease is positive (churn, defects).` }, { name: "html-data-app-embedding", description: "Embed an in-package HTML data app into a host page or another application, including auto-sizing and auth. Read when embedding a Publisher page via Publisher.embed.", body: '# Embedding an HTML Data App\n\n> `Publisher.embed(selector, { src })` drops a package page into a host page as a sandboxed, auto-resizing iframe. Same-origin embeds authenticate with the browser\'s cookies; cross-origin embeds need a signed token.\n\n## The host-page pattern\n\n```html\n<script src="https://your-publisher/sdk/publisher.js"></script>\n<div id="dashboard"></div>\n<script>\n  const handle = Publisher.embed("#dashboard", {\n    src: "https://your-publisher/environments/demo/packages/sales/index.html",\n  });\n  // handle.destroy() removes the iframe and detaches its listeners.\n</script>\n```\n\n`embed(selector, options)` returns `{ iframe, destroy() }`. Options: `src` (required), `token` (a signed token for cross-origin auth, appended as `embed_token`), `height` (omit to auto-size; a number is treated as pixels), and `allow` (the iframe permissions policy).\n\n## Sizing and the resize contract\n\nOmit `height` and the frame auto-sizes. The embedded page measures its real content height and posts a `publisher:resize` message to the host, which resizes the iframe and accepts that message only from the iframe it created. You write none of this; it ships in `/sdk/publisher.js`, so the embedded page only has to load that script.\n\nDo not rely on `body { min-height: 100vh }` to drive the frame height. The runtime deliberately measures the content\'s bottom edge, not the viewport, to avoid a grow-forever loop.\n\n## Auth\n\n- Same-origin or same-tenant: pass no token. The browser\'s cookies authenticate the iframe.\n- Cross-origin: mint a short-lived signed token server-side and pass it as `options.token`. The runtime appends it to the iframe URL as `embed_token`; the embedded page must read it (from `location.search`) and call `Publisher.setToken(token)`. Because it rides in the URL, it can land in browser history, Referer headers, and server logs, so keep it short-lived and scoped to that one embed, and never put a long-lived or admin token in client HTML.\n\n## Guardrails (v1)\n\n- The iframe is sandboxed (`allow-scripts allow-same-origin allow-forms`). Design for that: no top-level navigation, no popups.\n- Embedded author JavaScript runs with the viewing user\'s data authority, so treat everything under `public/` as strictly first-party code: do not load untrusted third-party scripts, and do not move query results off to other hosts. Tighter per-embed isolation is planned.' }, { name: "html-data-app-runtime", description: "Write the JavaScript that drives an in-package HTML data app, calling Publisher.query, building queries from filter state, and handling results and errors. Read before writing the page's data code.", body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"carriers.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName`, `filterParams` (values for the model\'s legacy `#(filter)` source filters; `Publisher.query` does not forward Malloy `given:` values), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese run against the example `carriers` package (source `carriers`; views `by_letter`, `by_size_bucket`, `kpis`). Swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("carriers.malloy", "run: carriers -> by_letter");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n  const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n  const parts = [];\n  if (state.letter) parts.push(`letter = \'${q(state.letter)}\'`);\n  if (state.bucket) parts.push(`size_bucket = \'${q(state.bucket)}\'`);\n  return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n  "carriers.malloy",\n  `run: carriers -> by_letter + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input. A data-app page has no clean server-side parameterization for arbitrary input today: `Publisher.query` forwards `opts.filterParams` (the deprecated `#(filter)` source-filter API), not Malloy `given:` values. So constrain the input to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("carriers.malloy", "run: carriers -> kpis");\nel.textContent = kpis.total;   // the result is an array; kpis.total, not rows.total\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [byLetter, byBucket, kpisRows] = await Promise.all([\n  Publisher.query("carriers.malloy", "run: carriers -> by_letter"),\n  Publisher.query("carriers.malloy", "run: carriers -> by_size_bucket"),\n  Publisher.query("carriers.malloy", "run: carriers -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n  ```js\n  // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n  // date to a "YYYY-MM" string; monthLabel formats it for display.\n  // target may not cover every actual month; an absent month stays null, never 0.\n  const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n  const data = actualRows.map((r) => ({\n    label: monthLabel(r.order_month),\n    actual: Number(r.revenue),\n    target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n  }));\n  ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("carriers.malloy", "run: carriers -> by_letter")\n  .then((rows) => {\n    if (!rows.length) { el.textContent = "No data."; return; }\n    render(rows);\n  })\n  .catch((err) => {\n    el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n  });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("carriers.malloy", "run: carriers -> by_letter");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The shipped example renders rows with a plain chart library instead, so this path is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: letter` gives a `letter` column; `aggregate: n is count()` gives an `n` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"carriers.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400, required parameter | The model marks a runtime parameter (given) as required. Supply it via `opts.filterParams`, or pass `bypassFilters: true` only if you are a trusted caller. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |' }, { name: "html-data-apps", description: "Build or modify an in-package HTML data app for a Malloy Publisher package (a public/ directory the package serves). Use when the user wants a hand-authored HTML dashboard or web page backed by a package's Malloy models, with no build step.", body: `# In-Package HTML Data Apps
+
+> A package becomes a web app by adding a \`public/\` directory. Publisher serves those files and gives the page \`Publisher.query(...)\` to run Malloy against the package's models. No build step, no npm, no framework.
+
+## When this is the right tool
+
+| The user wants | Use |
+|---|---|
+| A hand-authored HTML/JS dashboard, no toolchain | this skill (an HTML data app) |
+| A React app with managed components | the Publisher React SDK (out of scope here) |
+| An analyst notebook with charts | a Malloy notebook (\`.malloynb\`) |
+| Point-and-click exploration, no code | the Publisher Explorer |
+
+Pick an HTML data app when the user wants full control of the markup and only plain web files.
+
+## Package anatomy
+
+\`\`\`
+my-package/
+  publisher.json        # name, version, description
+  carriers.malloy       # the model(s), stays private
+  carriers.parquet      # data, stays private
+  public/               # ONLY this directory is web-served
+    index.html
+    app.js
+\`\`\`
+
+Only \`public/\` is reachable over the web, at \`/environments/<env>/packages/<pkg>/<file>\`. Models, data, and \`publisher.json\` are private and reached only through the query API, which still applies the model's filters, access modifiers, and authorize rules. There is no flag to set: a \`public/\` directory is what makes a package an app.
+
+## Build sequence
+
+The agent orchestrates these. Each query and chart step hands off to a focused skill.
+
+1. READ THE MODEL FIRST. Get the model's real source and view names, through your environment's context tool if it has one, or by opening the \`.malloy\` file directly. Never guess field or view names.
+2. SCAFFOLD the package (template below).
+3. WRITE THE QUERIES with \`skill:html-data-app-runtime\`. Validate each before pasting it into the page, using whatever query tool your environment provides or a running Publisher (see \`skill:html-data-app-runtime\`). Malloy syntax questions go to \`skill:malloy-queries\`.
+4. CHOOSE CHARTS with \`skill:malloy-charts\` when rendering through \`<malloy-render>\`; otherwise it is your own chart library drawing the returned rows. Vendor any chart library into \`public/\` and load it locally, not from a CDN, because embedded author JavaScript runs with the viewing user's data authority. (The shipped example loads its chart library from a CDN for brevity; a published or embedded app should vendor it.)
+5. EMBED (optional) with \`skill:html-data-app-embedding\`.
+6. PREVIEW with the local authoring loop (below).
+7. VERIFY before you call it done (see "What 'done' means" below). This step is not optional.
+
+The scaffold in step 2 only proves the wiring. It is the start, not the deliverable. What you ship is a production app that meets the recipe below.
+
+## What "done" means (production recipe)
+
+A data app you can defend has all of these. Build to this list, not to the scaffold.
+
+- **Real names, never guessed.** Every source, view, and field name comes from the model you read in step 1. A name you derived or assumed is a bug waiting to surface as an empty tile.
+- **Modular, not one inline blob.** Split the page into modules per \`skill:html-data-app-runtime\` (pure formatting helpers, a chart layer, your tile/query definitions as data, a thin entry point). One source of truth for each tile's model/source/view, no parallel maps that drift.
+- **Every tile handles loading, empty, and error on its own.** One failing query must not blank the page. (\`skill:html-data-app-runtime\`.)
+- **Defensible numbers.** Missing ≠ zero (omit the point, don't plot a fake 0); show the latest non-null value for "current"; guard division with \`nullif\`; convert units explicitly. (\`skill:html-data-app-runtime\`.)
+- **Visible assumptions.** When you assume something (two sources joined by month, an in-month proxy that differs from a certified definition) or a metric is incomplete, say so *in the app*: a caption, a footnote, a placeholder card with the reason. The non-technical user cannot see your reasoning; bury a caveat and you have misled them. Don't silently drop a metric you couldn't model. Show a placeholder that names what's missing and why.
+- **Looks decent.** Give it real layout, type, and color: a styled card grid with a clear hierarchy, not raw unstyled tables.
+- **Vendored libraries.** Chart and helper libraries live in \`public/\`, loaded locally (step 4).
+
+### Verify before you call it done
+
+You are building for someone who cannot tell a correct dashboard from a broken one. Verification is your job, not theirs.
+
+- **Validate every query against the model before wiring it in** (step 3): confirm it compiles and that the column names match what your render code reads.
+- **Load the finished page and confirm every tile shows real numbers**: not stuck on "Loading…", not an error, not an empty state you didn't intend. In a headless browser, wait on \`load\` plus a content selector, not network idle (\`publisher.js\` holds an SSE stream open; see \`skill:html-data-app-runtime\`).
+- **Unit-test any non-trivial pure logic** (a month-join, a de-cumulation, a unit conversion). Keep that logic in DOM-free helpers so \`node --test\` can cover it, and run it.
+
+## Minimal scaffold
+
+\`publisher.json\` at the package root:
+
+\`\`\`json
+{ "name": "my-package", "version": "0.0.1", "description": "..." }
+\`\`\`
+
+\`public/index.html\` is a NEW file you create (make the \`public/\` directory if it does not exist). Load the runtime root-relative, then query. The examples below use the shipped \`carriers\` package; swap in your own model and a view it defines.
+
+Start with the smallest page that proves the wiring, dumping the rows:
+
+\`\`\`html
+<!doctype html>
+<title>My dashboard</title>
+<pre id="out"></pre>
+<script src="/sdk/publisher.js"></script>
+<script>
+  Publisher.query("carriers.malloy", "run: carriers -> by_letter").then((rows) => {
+    document.getElementById("out").textContent = JSON.stringify(rows, null, 2);
+  });
+</script>
+\`\`\`
+
+Then render the rows. This page builds a table from whatever columns the view returns, so it does not depend on the exact field names:
+
+\`\`\`html
+<!doctype html>
+<title>Carriers by letter</title>
+<table id="t"><thead></thead><tbody></tbody></table>
+<script src="/sdk/publisher.js"></script>
+<script>
+  Publisher.query("carriers.malloy", "run: carriers -> by_letter").then((rows) => {
+    const t = document.getElementById("t");
+    if (!rows.length) { t.textContent = "No rows."; return; }
+    const cols = Object.keys(rows[0]);
+    const headRow = t.tHead.insertRow();
+    for (const c of cols) {
+      const th = document.createElement("th");
+      th.textContent = c;
+      headRow.appendChild(th);
+    }
+    for (const r of rows) {
+      const tr = t.tBodies[0].insertRow();
+      for (const c of cols) tr.insertCell().textContent = r[c];
+    }
+  });
+</script>
+\`\`\`
+
+Build row content with \`textContent\`, not \`innerHTML\` with model values: an \`innerHTML\` table renders any markup a value contains. This is the HTML-output side of the don't-trust-interpolated-values rule that \`skill:html-data-app-runtime\` applies to Malloy query strings.
+
+Two invariants break a page most often:
+
+- **The file must live under \`public/\`.** Publisher serves only \`public/\`, so a page written anywhere else (for example \`/tmp\`) is never reachable at \`/environments/<env>/packages/<pkg>/<file>\`.
+- **The script src must be the root-relative \`/sdk/publisher.js\`**, not a relative path.
+
+A third gotcha: the first argument to \`Publisher.query\` is the model FILE path (\`"carriers.malloy"\`), not the source name.
+
+## Authoring loop and publishing
+
+Authoring happens locally, then you publish. These are two stages.
+
+### Author locally (with live reload)
+
+Run a local Publisher from the directory that holds your \`publisher.config.json\` and package folder(s):
+
+\`\`\`sh
+npx @malloy-publisher/server --server_root . --port 4000 --watch-env <env>
+\`\`\`
+
+\`--watch-env <env>\` (or \`PUBLISHER_WATCH=<env>\`) mounts that environment's local-dir packages in place (a symlink, not a copy) and watches them: editing a \`.malloy\` recompiles the package, and editing a \`public/\` file live-reloads any open page over an SSE stream. Nothing to wire in the page. The app is served at \`http://localhost:4000/environments/<env>/packages/<pkg>/index.html\`.
+
+\`publisher.config.json\` (at \`--server_root\`) declares the environment, its packages, and its connections:
+
+\`\`\`json
+{
+  "frozenConfig": false,
+  "environments": [
+    {
+      "name": "<env>",
+      "packages": [{ "name": "<pkg>", "location": "./<pkg>" }],
+      "connections": []
+    }
+  ]
+}
+\`\`\`
+
+A local package uses a filesystem \`location\` (\`"./<pkg>"\`, relative to \`--server_root\`); a remote one uses a GitHub \`tree\` URL. If one model in the package fails to compile, the **whole package** fails to load, so a stray notebook/model error blanks every tile. (Common one: a \`.malloynb\` whose cells each \`import "x.malloy"\`, the notebook compiles as one batch, so the repeated import errors \`Cannot redefine 'x'\`. Import once in the first cell.)
+
+### Publishing
+
+Publishing an app is publishing its package: get the package into publishable shape and hand it to your Publisher host's deploy path (see \`skill:malloy-publish\`). A deployed package serves its \`public/\` app the same way a local one does, at \`/environments/<env>/packages/<pkg>/<file>\`. A deployed environment has no \`--watch-env\` live reload, so the loop there is author, publish, then view.` }, { name: "lookml-review", description: "Analyze LookML files as prior art for Malloy modeling. Used during Step 1 (DISCOVER) when .lkml files are present. Coordinates reference files that extract business logic, relationships, and curation decisions. Works with or without a database connection.", body: `# LookML Review
 
 > **Purpose:** Evaluate a LookML project as prior art for building a Malloy semantic model. This skill coordinates the LookML adapter. The implementation lives in reference files under \`reference/\`.
 
@@ -267921,7 +268293,9 @@ class MaterializationService {
     this.runInBackground(created.id, (signal) => this.runBuild(created.id, environmentName, packageName, {
       sourceNames: options.sourceNames,
       forceRefresh,
-      buildInstructions
+      buildInstructions,
+      referenceManifest: options.referenceManifest,
+      strictUpstreams: options.strictUpstreams
     }, signal));
     return created;
   }
@@ -267946,12 +268320,12 @@ class MaterializationService {
       let carried;
       if (orchestrated) {
         instructions = opts.buildInstructions;
-        carried = {};
+        carried = this.referenceManifestToEntries(opts.referenceManifest);
       } else {
         const priorEntries = opts.forceRefresh ? {} : await this.getMostRecentManifestEntries(environmentId, packageName, id);
         ({ instructions, carried } = this.deriveSelfInstructions(compiled, opts.sourceNames, priorEntries));
       }
-      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal);
+      const entries = await this.executeInstructedBuild(compiled, instructions, carried, signal, opts.strictUpstreams ?? false);
       const sourcesBuilt = instructions.length;
       const sourcesReused = Object.keys(carried).length;
       const durationMs = Date.now() - startedAt;
@@ -268010,6 +268384,16 @@ class MaterializationService {
     }
     return { instructions, carried };
   }
+  referenceManifestToEntries(referenceManifest) {
+    const entries = {};
+    for (const ref of referenceManifest ?? []) {
+      entries[ref.sourceEntityId] = {
+        sourceEntityId: ref.sourceEntityId,
+        physicalTableName: ref.physicalTableName
+      };
+    }
+    return entries;
+  }
   async getMostRecentManifestEntries(environmentId, packageName, excludeId) {
     const list = await this.repository.listMaterializations(environmentId, packageName);
     for (const m of list) {
@@ -268062,7 +268446,7 @@ class MaterializationService {
       }
     }
   }
-  async executeInstructedBuild(compiled, instructions, seedEntries, signal) {
+  async executeInstructedBuild(compiled, instructions, seedEntries, signal, strict = false) {
     const { graphs, sources, connectionDigests, connections } = compiled;
     const bySourceID = new Map;
     const bySourceEntityId = new Map;
@@ -268073,6 +268457,7 @@ class MaterializationService {
       bySourceEntityId.set(instruction.sourceEntityId, instruction);
     }
     const manifest = new Manifest;
+    manifest.strict = strict;
     const entries = {};
     for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
       if (entry.physicalTableName) {