@malloy-publisher/server 0.5.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. package/dist/app/api-doc.yaml +125 -59
  2. package/dist/app/assets/{EnvironmentPage-B02geyxh.js → EnvironmentPage-ufeWfqnK.js} +1 -1
  3. package/dist/app/assets/{HomePage-BdKzZEto.js → HomePage-CN_FgmtD.js} +1 -1
  4. package/dist/app/assets/{LightMode-BRAs1k98.js → LightMode-BFbC4W45.js} +1 -1
  5. package/dist/app/assets/{MainPage-D9XZbGvk.js → MainPage-BcCyzakx.js} +1 -1
  6. package/dist/app/assets/{ModelPage-B-U47DzF.js → ModelPage-wHHnxoih.js} +2 -2
  7. package/dist/app/assets/{PackagePage-CDp9A3_k.js → PackagePage-DwdO-2dp.js} +1 -1
  8. package/dist/app/assets/{RenderedResult-CTD2Ej2S.es-BZsZAiwo.js → RenderedResult-CYyhQekT.es-DBHF8EwO.js} +2 -2
  9. package/dist/app/assets/{RouteError-CSsLG99c.js → RouteError-Blt6a2Za.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-C4u5l49i.js → ThemeEditorPage-Cb-2YbfG.js} +1 -1
  11. package/dist/app/assets/{ToggleButtonGroup-IBhUYkv_.js → ToggleButtonGroup-BuhURn8v.js} +1 -1
  12. package/dist/app/assets/{index-DOXSuS-a.js → index-BELjGKhI.js} +1 -1
  13. package/dist/app/assets/{index--DAcpR5Y.js → index-BJyLY8XQ.js} +1 -1
  14. package/dist/app/assets/{index-BKNRDF5y.js → index-DCizS2BO.js} +1 -1
  15. package/dist/app/assets/{index-Bd7WYNw_.js → index-Dz4y_2PE.js} +4 -4
  16. package/dist/app/assets/{index-CshLRozl.js → index-kt4oTEGz.js} +1 -1
  17. package/dist/app/assets/{index.es-Bj77sE9l.js → index.es-u5PRRTzT.js} +4 -4
  18. package/dist/app/index.html +1 -1
  19. package/dist/cpufeatures-1yrn0vtw.node +0 -0
  20. package/dist/package_load_worker.mjs +747 -357
  21. package/dist/server.mjs +1450 -628
  22. package/dist/{sshcrypto-xqan60jb.node → sshcrypto-cqb52edz.node} +0 -0
  23. package/package.json +1 -1
package/dist/server.mjs CHANGED
@@ -175849,7 +175849,7 @@ import os from "os";
175849
175849
  function normalizeModelPath(p) {
175850
175850
  return p.replace(/\\/g, "/").replace(/^\.\//, "");
175851
175851
  }
175852
- var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
175852
+ var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", INDEX_MODEL_NAME = "index.malloy", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
175853
175853
  var init_constants = __esm(() => {
175854
175854
  TEMP_DIR_PATH = os.tmpdir();
175855
175855
  });
@@ -232535,12 +232535,23 @@ var require_bcrypt_pbkdf = __commonJS((exports, module) => {
232535
232535
  };
232536
232536
  });
232537
232537
 
232538
+ // ../../node_modules/cpu-features/build/Release/cpufeatures.node
232539
+ var require_cpufeatures = __commonJS((exports, module) => {
232540
+ module.exports = __require("./cpufeatures-1yrn0vtw.node");
232541
+ });
232542
+
232543
+ // ../../node_modules/cpu-features/lib/index.js
232544
+ var require_lib8 = __commonJS((exports, module) => {
232545
+ var binding = require_cpufeatures();
232546
+ module.exports = binding.getCPUInfo;
232547
+ });
232548
+
232538
232549
  // ../../node_modules/ssh2/lib/protocol/constants.js
232539
232550
  var require_constants42 = __commonJS((exports, module) => {
232540
232551
  var crypto3 = __require("crypto");
232541
232552
  var cpuInfo;
232542
232553
  try {
232543
- cpuInfo = (()=>{throw new Error("Cannot require module "+"cpu-features");})()();
232554
+ cpuInfo = require_lib8()();
232544
232555
  } catch {}
232545
232556
  var { bindingAvailable, CIPHER_INFO, MAC_INFO } = require_crypto();
232546
232557
  var eddsaSupported = (() => {
@@ -233139,7 +233150,7 @@ var require_utils92 = __commonJS((exports, module) => {
233139
233150
 
233140
233151
  // ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
233141
233152
  var require_sshcrypto = __commonJS((exports, module) => {
233142
- module.exports = __require("./sshcrypto-xqan60jb.node");
233153
+ module.exports = __require("./sshcrypto-cqb52edz.node");
233143
233154
  });
233144
233155
 
233145
233156
  // ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
@@ -247183,7 +247194,7 @@ var require_keygen = __commonJS((exports, module) => {
247183
247194
  });
247184
247195
 
247185
247196
  // ../../node_modules/ssh2/lib/index.js
247186
- var require_lib8 = __commonJS((exports, module) => {
247197
+ var require_lib9 = __commonJS((exports, module) => {
247187
247198
  var {
247188
247199
  AgentProtocol,
247189
247200
  BaseAgent,
@@ -247366,7 +247377,7 @@ function openSshProxy(ssh, target) {
247366
247377
  var import_ssh2, ALLOW_UNVERIFIED_SSH_HOST_KEY_ENV = "PUBLISHER_ALLOW_UNVERIFIED_SSH_HOST_KEY", TRUTHY_ENV_VALUES, SSH_CONNECT_TIMEOUT_MS = 15000, SSH_KEEPALIVE_INTERVAL_MS = 15000;
247367
247378
  var init_proxy = __esm(() => {
247368
247379
  init_logger();
247369
- import_ssh2 = __toESM(require_lib8(), 1);
247380
+ import_ssh2 = __toESM(require_lib9(), 1);
247370
247381
  TRUTHY_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
247371
247382
  });
247372
247383
 
@@ -267096,6 +267107,29 @@ var require_lunr = __commonJS((exports, module) => {
267096
267107
  })();
267097
267108
  });
267098
267109
 
267110
+ // src/service/authorize_routes.ts
267111
+ function canonicalAuthorizeRoute(route) {
267112
+ switch (route) {
267113
+ case ACCESS_FILTER_ROUTE:
267114
+ return ACCESS_FILTER_ROUTE;
267115
+ case AUTHORIZE_ROUTE:
267116
+ return AUTHORIZE_ROUTE;
267117
+ default:
267118
+ return;
267119
+ }
267120
+ }
267121
+ var AUTHORIZE_ROUTE = "authorize", ACCESS_FILTER_ROUTE = "access_filter", RECOGNIZED_AUTHORIZE_SPELLINGS, CANONICAL_AUTHORIZE_ROUTES;
267122
+ var init_authorize_routes = __esm(() => {
267123
+ RECOGNIZED_AUTHORIZE_SPELLINGS = [
267124
+ ACCESS_FILTER_ROUTE,
267125
+ AUTHORIZE_ROUTE
267126
+ ];
267127
+ CANONICAL_AUTHORIZE_ROUTES = [
267128
+ ACCESS_FILTER_ROUTE,
267129
+ AUTHORIZE_ROUTE
267130
+ ];
267131
+ });
267132
+
267099
267133
  // src/service/authorize.ts
267100
267134
  import { payloadOf, routeOf } from "@malloydata/malloy";
267101
267135
  function noteRoute(text) {
@@ -267104,8 +267138,61 @@ function noteRoute(text) {
267104
267138
  function notePayload(text) {
267105
267139
  return payloadOf({ value: text.trimStart() }) ?? "";
267106
267140
  }
267141
+ function nearMissRouteNames(route) {
267142
+ switch (route) {
267143
+ case AUTHORIZE_ROUTE:
267144
+ return [
267145
+ "authorize",
267146
+ "source_authorize",
267147
+ "source-authorize",
267148
+ "sourceauthorize",
267149
+ "authorize_source",
267150
+ "authorize-source",
267151
+ "authorizesource"
267152
+ ];
267153
+ case ACCESS_FILTER_ROUTE:
267154
+ return [
267155
+ "access_filter",
267156
+ "access-filter",
267157
+ "accessfilter",
267158
+ "filter_access",
267159
+ "filter-access",
267160
+ "filteraccess",
267161
+ "row_authorize",
267162
+ "row-authorize",
267163
+ "rowauthorize",
267164
+ "authorize_row",
267165
+ "authorize-row",
267166
+ "authorizerow"
267167
+ ];
267168
+ default:
267169
+ return [route];
267170
+ }
267171
+ }
267172
+ function nearMissWordAlternation(route) {
267173
+ const names = nearMissRouteNames(route);
267174
+ if (names.length === 0) {
267175
+ throw new Error(`No near-miss spellings for authorize route \`${route}\` — an empty ` + "alternation matches every malformed-prefix note, which would " + "fail package loads over annotations nobody wrote.");
267176
+ }
267177
+ return names.map((name) => name.replace(REGEXP_SPECIAL, "\\$&")).join("|");
267178
+ }
267179
+ function motlyAuthorizePayloadPattern(route) {
267180
+ return new RegExp(`^[ \\t]*[([{<][ \\t]*(?:${nearMissWordAlternation(route)})[ \\t]*[)\\]}>]`, "iu");
267181
+ }
267182
+ function malformedAuthorizeAttemptPattern(route) {
267183
+ return new RegExp(`^##?\\|?[ \\t]*[([{<]?[ \\t]*(?:${nearMissWordAlternation(route)})`, "iu");
267184
+ }
267107
267185
  function authorizeNoteContent(text) {
267108
- return noteRoute(text) === AUTHORIZE_ROUTE ? notePayload(text) : undefined;
267186
+ const route = canonicalAuthorizeRoute(noteRoute(text));
267187
+ if (route === undefined)
267188
+ return;
267189
+ return { route, content: notePayload(text) };
267190
+ }
267191
+ function authorizeAnnotationRoute(text) {
267192
+ return authorizeNoteContent(text)?.route;
267193
+ }
267194
+ function hasCallerAuthorizeAnnotation(callerText) {
267195
+ return AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText);
267109
267196
  }
267110
267197
  function assertNoCallerAuthorizeAnnotation(callerText) {
267111
267198
  if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
@@ -267115,37 +267202,59 @@ function assertNoCallerAuthorizeAnnotation(callerText) {
267115
267202
  function containsAuthorizeAnnotationTag(texts) {
267116
267203
  return texts.some((text) => authorizeNoteContent(text) !== undefined);
267117
267204
  }
267118
- function collectAuthorizeNearMisses(texts) {
267205
+ function collectAuthorizeNearMisses(texts, route) {
267206
+ const motlyPattern = motlyAuthorizePayloadPattern(route);
267207
+ const malformedPattern = malformedAuthorizeAttemptPattern(route);
267208
+ const nearMissNames = new Set(nearMissRouteNames(route).map((name) => name.toLowerCase()));
267119
267209
  const found = [];
267120
267210
  for (const text of texts) {
267121
267211
  const trimmed = text.trimStart();
267122
- const route = noteRoute(trimmed);
267123
- if (route === AUTHORIZE_ROUTE)
267212
+ const noted = noteRoute(trimmed);
267213
+ if (noted === route)
267124
267214
  continue;
267125
- const nearMiss = route === undefined ? MALFORMED_AUTHORIZE_ATTEMPT.test(trimmed) : route === "" ? MOTLY_AUTHORIZE_PAYLOAD.test(notePayload(trimmed)) : route.toLowerCase() === AUTHORIZE_ROUTE;
267215
+ const nearMiss = noted === undefined ? malformedPattern.test(trimmed) : noted === "" ? motlyPattern.test(notePayload(trimmed)) : nearMissNames.has(noted.toLowerCase());
267126
267216
  if (!nearMiss)
267127
267217
  continue;
267128
267218
  found.push(trimmed.split(/[\r\n]/, 1)[0]);
267129
267219
  }
267130
267220
  return found;
267131
267221
  }
267222
+ function collectAuthorizeNearMissesAllRoutes(texts) {
267223
+ const materialized = [...texts];
267224
+ return RECOGNIZED_AUTHORIZE_SPELLINGS.flatMap((route) => collectAuthorizeNearMisses(materialized, route));
267225
+ }
267226
+ function guessedNearMissRoute(text) {
267227
+ const lower = text.toLowerCase();
267228
+ let best = ACCESS_FILTER_ROUTE;
267229
+ let bestLength = 0;
267230
+ for (const route of RECOGNIZED_AUTHORIZE_SPELLINGS) {
267231
+ for (const name of nearMissRouteNames(route)) {
267232
+ if (lower.includes(name) && name.length > bestLength) {
267233
+ best = route;
267234
+ bestLength = name.length;
267235
+ }
267236
+ }
267237
+ }
267238
+ return best;
267239
+ }
267132
267240
  function assertNoAuthorizeNearMisses(found) {
267133
267241
  if (found.length === 0)
267134
267242
  return;
267135
267243
  const unique = [...new Set(found)];
267136
267244
  throw new ModelCompilationError({
267137
- message: `These annotations are not \`authorize\` gates and nothing enforces ` + `them:
267138
- ${unique.map((t) => ` - \`${t}\``).join(`
267245
+ message: `These annotations are never enforced:
267246
+ ${unique.map((t) => ` - \`${t}\` (meant \`#(${guessedNearMissRoute(t)})\`?)`).join(`
267139
267247
  `)}
267140
- ` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\` (or \`##(authorize)\`, or the block form ` + `\`#|(authorize)\`) reaches the authorize route — a space after the ` + `\`#\`, spaces inside the brackets, or anything trailing the closing ` + `bracket makes it a plain tag Malloy hands to something else. Write ` + `\`#(authorize) <expression>\` on its own line directly above the ` + `\`source:\` statement you mean to protect (unquoted — the quoted ` + `string form is retired). This is refused rather than interpreted: ` + `guessing at the ` + `intent would let publisher start enforcing a filter on a package that ` + `has been serving every row.`
267248
+ ` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\` and \`#(access_filter)\` reach an access-control ` + `route (in their \`##\` and ` + `block-form \`#|(...)\` spellings too). A space after the \`#\`, spaces ` + `inside the brackets, a mis-hyphenated or transposed word, or anything ` + `trailing the closing bracket makes it a plain tag Malloy hands to ` + `something else. Write the tag named above, unquoted, on its own line ` + `directly above the \`source:\` statement you mean to protect (the ` + `quoted string form is retired). This is refused rather than ` + `interpreted: guessing at the intent would let publisher start ` + `enforcing a filter on a package that has been serving every row.`
267141
267249
  });
267142
267250
  }
267143
267251
  function describeMisplacedAuthorizeAnnotation(f) {
267144
- if (f.kind === "query")
267145
- return `on query "${f.name}"`;
267146
267252
  if (f.kind === "file")
267147
- return "at the file level (`##(authorize)`)";
267148
- return `on field "${f.fieldName}" of source "${f.name}"`;
267253
+ return `at the file level (\`##(${f.route})\`)`;
267254
+ const tag = `\`#(${f.route})\``;
267255
+ if (f.kind === "query")
267256
+ return `on query "${f.name}" (${tag})`;
267257
+ return `on field "${f.fieldName}" of source "${f.name}" (${tag})`;
267149
267258
  }
267150
267259
  function assertNoMisplacedAuthorizeAnnotations(found) {
267151
267260
  if (found.length === 0)
@@ -267153,9 +267262,9 @@ function assertNoMisplacedAuthorizeAnnotations(found) {
267153
267262
  const positions = found.map((f) => ` - ${describeMisplacedAuthorizeAnnotation(f)}`).join(`
267154
267263
  `);
267155
267264
  throw new ModelCompilationError({
267156
- message: `An \`#(authorize)\` annotation is never enforced at:
267265
+ message: `An authorize annotation is never enforced at:
267157
267266
  ${positions}
267158
- ` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base. File-level \`##(authorize)\` is deprecated and no longer ` + `enforced anywhere, so it always lands here: declare \`#(authorize)\` ` + `on each \`source:\` it was meant to protect instead. Every other ` + `position above should move to the \`source:\` statement it is meant ` + `to protect.`
267267
+ ` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base. A file-level annotation (\`##(authorize)\` / ` + `\`##(access_filter)\`) is deprecated and no longer enforced ` + `anywhere, so it always lands here: declare the same tag on each ` + `\`source:\` it was meant to protect instead. Every other ` + `position above should move — with the same tag named beside it — ` + `to the \`source:\` statement it is meant ` + `to protect.`
267159
267268
  });
267160
267269
  }
267161
267270
  function referencedGivenNames(expr) {
@@ -267192,7 +267301,7 @@ function liftProbeFilterCondition(prepared, label, filterText) {
267192
267301
  return lifted;
267193
267302
  }
267194
267303
  function gateFilterText(exprs) {
267195
- return exprs.map((e) => `(${e})`).join(" or ");
267304
+ return exprs.map((e) => `(${e})`).join(" and ");
267196
267305
  }
267197
267306
  async function liftRowLevelCondition(compiler, sourceName, exprs) {
267198
267307
  const filterText = gateFilterText(exprs);
@@ -267202,7 +267311,7 @@ async function liftRowLevelCondition(compiler, sourceName, exprs) {
267202
267311
  async function validateAuthorizeProbes(compiler, options) {
267203
267312
  const ownNotesOf = options.authorizeOwnNotes ?? new Map;
267204
267313
  for (const [sourceName, groups] of options.authorizeMap ?? []) {
267205
- for (const exprs of groups) {
267314
+ for (const { route, exprs } of groups) {
267206
267315
  if (exprs.length === 0)
267207
267316
  continue;
267208
267317
  let condition;
@@ -267210,17 +267319,17 @@ async function validateAuthorizeProbes(compiler, options) {
267210
267319
  condition = await liftRowLevelCondition(compiler, sourceName, exprs);
267211
267320
  } catch (err) {
267212
267321
  const detail = err instanceof Error ? err.message : String(err);
267213
- const ownNotes = ownNotesOf.get(sourceName) ?? [];
267322
+ const ownNotes = ownNotesOf.get(sourceName)?.get(route) ?? [];
267214
267323
  if (ownNotes.length === 0) {
267215
267324
  options.onRowLevelGateRejected?.("entry_point_unexpressible");
267216
267325
  options.onRowLevelGateUnexpressible?.(sourceName, detail);
267217
267326
  continue;
267218
267327
  }
267219
267328
  throw new ModelCompilationError({
267220
- message: `Invalid #(authorize) annotation on source "${sourceName}" ` + `[${exprs.join(" | ")}]: ${detail}`
267329
+ message: `Invalid #(${route}) annotation on source "${sourceName}" ` + `[${exprs.join(" and ")}]: ${detail}`
267221
267330
  });
267222
267331
  }
267223
- options.onOwnRowLevelConditionCompiled?.(sourceName, condition);
267332
+ options.onOwnRowLevelConditionCompiled?.(sourceName, condition, route);
267224
267333
  }
267225
267334
  }
267226
267335
  }
@@ -267234,59 +267343,41 @@ function unquoteLegacyGatePayload(payload) {
267234
267343
  return trimmed[0] === "'" ? inner.replace(SINGLE_QUOTE_ESCAPE, "$1") : inner.replace(DOUBLE_QUOTE_ESCAPE, "$1");
267235
267344
  }
267236
267345
  function parseAuthorizeAnnotation(annotation) {
267237
- const content = authorizeNoteContent(annotation);
267238
- if (content === undefined)
267346
+ const parsed = authorizeNoteContent(annotation);
267347
+ if (parsed === undefined)
267239
267348
  return null;
267240
- const trimmed = content.trim();
267349
+ const trimmed = parsed.content.trim();
267241
267350
  if (trimmed.length === 0) {
267242
267351
  throw new Error("authorize annotation has an empty expression body");
267243
267352
  }
267244
- return trimmed;
267353
+ return { route: parsed.route, expr: trimmed };
267245
267354
  }
267246
267355
  function collectAuthorizeExprs(annotations) {
267247
267356
  const exprs = [];
267248
267357
  for (const annotation of annotations) {
267249
- const expr = parseAuthorizeAnnotation(annotation);
267250
- if (expr !== null) {
267251
- exprs.push(expr);
267358
+ const parsed = parseAuthorizeAnnotation(annotation);
267359
+ if (parsed !== null) {
267360
+ exprs.push(parsed);
267252
267361
  }
267253
267362
  }
267254
267363
  return exprs;
267255
267364
  }
267256
- function findMultipleAuthorizeGates(authorizeOwnNotes) {
267257
- const found = [];
267258
- for (const [sourceName, notes] of authorizeOwnNotes) {
267259
- if (notes.length > 1) {
267260
- found.push({ sourceName, texts: notes.map((note) => note.text) });
267261
- }
267262
- }
267263
- return found;
267264
- }
267265
- function assertAtMostOneAuthorizeGate(found) {
267266
- if (found.length === 0)
267267
- return;
267268
- const positions = found.map(({ sourceName, texts }) => ` - source "${sourceName}" declares ${texts.length}:
267269
- ` + texts.map((t) => ` \`${t.trim().split(/[\r\n]/, 1)[0]}\``).join(`
267270
- `)).join(`
267271
- `);
267272
- throw new ModelCompilationError({
267273
- message: `A source may declare at most one \`#(authorize)\` annotation:
267274
- ${positions}
267275
- ` + `Combine multiple conditions into one expression with \`or\` instead of ` + `repeating the annotation, e.g. ` + "`#(authorize) $ROLE = 'admin' or org_id in $GROUPS`."
267276
- });
267365
+ function collectAuthorizeExprsForRoute(annotations, route) {
267366
+ return collectAuthorizeExprs(annotations).filter((e) => e.route === route).map((e) => e.expr);
267277
267367
  }
267278
267368
  function findLegacyStringGates(authorizeOwnNotes) {
267279
267369
  const found = [];
267280
- for (const [sourceName, notes] of authorizeOwnNotes) {
267370
+ for (const [sourceName, notesByRoute] of authorizeOwnNotes) {
267371
+ const notes = [...notesByRoute.values()].flat();
267281
267372
  const legacyNotes = notes.filter((note) => {
267282
267373
  const content = authorizeNoteContent(note.text);
267283
- return content !== undefined && isLegacyQuotedPayload(content);
267374
+ return content !== undefined && isLegacyQuotedPayload(content.content);
267284
267375
  });
267285
267376
  if (legacyNotes.length === 0)
267286
267377
  continue;
267287
- const exprs = collectAuthorizeExprs(legacyNotes.map((note) => note.text));
267288
- if (exprs.length > 0) {
267289
- found.push({ sourceName, exprs });
267378
+ const gates = legacyNotes.flatMap((note) => collectAuthorizeExprs([note.text]));
267379
+ if (gates.length > 0) {
267380
+ found.push({ sourceName, gates });
267290
267381
  }
267291
267382
  }
267292
267383
  return found;
@@ -267294,22 +267385,22 @@ function findLegacyStringGates(authorizeOwnNotes) {
267294
267385
  function assertNoLegacyStringGate(found) {
267295
267386
  if (found.length === 0)
267296
267387
  return;
267297
- const rewrites = found.flatMap(({ sourceName, exprs }) => exprs.map((expr) => ` - source "${sourceName}": replace \`#(authorize) ${expr}\`` + ` with
267298
- #(authorize) ${unquoteLegacyGatePayload(expr)}`)).join(`
267388
+ const rewrites = found.flatMap(({ sourceName, gates }) => gates.map(({ route, expr }) => ` - source "${sourceName}": replace \`#(${route}) ${expr}\`` + ` with
267389
+ #(${route}) ${unquoteLegacyGatePayload(expr)}`)).join(`
267299
267390
  `);
267300
267391
  throw new ModelCompilationError({
267301
- message: `The string form of \`#(authorize)\` (a Malloy-quoted expression on ` + `the \`source:\` line) is no longer accepted. Replace it with the ` + `unquoted expression, carried by an \`#(authorize)\` annotation on ` + `its own line directly above the \`source:\` line:
267392
+ message: `The string form of an authorize gate (a Malloy-quoted expression on ` + `the \`source:\` line) is no longer accepted. Replace it with the ` + `unquoted expression, carried by the same annotation on its own line ` + `directly above the \`source:\` line:
267302
267393
  ${rewrites}
267303
267394
  ` + `Test that the rewrite gates the same rows and check the row count ` + `matches what the string form served.`
267304
267395
  });
267305
267396
  }
267306
- var AUTHORIZE_ROUTE = "authorize", AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, MOTLY_AUTHORIZE_PAYLOAD, MALFORMED_AUTHORIZE_ATTEMPT, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
267397
+ var AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, REGEXP_SPECIAL, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
267307
267398
  var init_authorize = __esm(() => {
267308
267399
  init_errors();
267309
- AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*[([{<]?[ \t]*authorize(?=[)\]}>]|[ \t]|$)`;
267400
+ init_authorize_routes();
267401
+ AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*[([{<]?[ \t]*(?:(?:(?:row|source)[-_]?)?authorize|access[-_]?filter)(?=[)\]}>]|[ \t]|$)`;
267310
267402
  AUTHORIZE_ANNOTATION_ANYWHERE = new RegExp(AUTHORIZE_TAG_LIKE, "iu");
267311
- MOTLY_AUTHORIZE_PAYLOAD = /^[ \t]*[([{<][ \t]*authorize[ \t]*[)\]}>]/iu;
267312
- MALFORMED_AUTHORIZE_ATTEMPT = /^##?\|?[ \t]*[([{<]?[ \t]*authorize/iu;
267403
+ REGEXP_SPECIAL = /[.*+?^${}()|[\]\\]/g;
267313
267404
  GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
267314
267405
  SINGLE_QUOTE_ESCAPE = /\\(['\\])/g;
267315
267406
  DOUBLE_QUOTE_ESCAPE = /\\(["\\])/g;
@@ -267894,7 +267985,13 @@ function collectGivenRefs(value, into) {
267894
267985
  for (const child of Object.values(node))
267895
267986
  collectGivenRefs(child, into);
267896
267987
  }
267897
- function suggestGivenLookup(modelDef, authorizeBySource, surfaced) {
267988
+ function gateGivenSource(sources, name) {
267989
+ const source = sources.find((candidate) => candidate.name === name);
267990
+ if (!source)
267991
+ return;
267992
+ return [...source.authorize ?? [], ...source.accessFilter ?? []];
267993
+ }
267994
+ function suggestGivenLookup(modelDef, gatesBySource, surfaced) {
267898
267995
  const registry = modelDef.givens ?? {};
267899
267996
  const bySource = new Map;
267900
267997
  const byQuery = new Map;
@@ -267903,7 +268000,7 @@ function suggestGivenLookup(modelDef, authorizeBySource, surfaced) {
267903
268000
  const name = obj.as || obj.name;
267904
268001
  const refs = new Set;
267905
268002
  collectGivenRefs(obj.filterList, refs);
267906
- for (const expr of authorizeBySource(name) ?? []) {
268003
+ for (const expr of gatesBySource(name) ?? []) {
267907
268004
  for (const given of referencedGivenNames(expr))
267908
268005
  refs.add(given);
267909
268006
  }
@@ -268002,7 +268099,7 @@ function positiveInteger(value) {
268002
268099
  function normalizeTileExpression(tile) {
268003
268100
  return tile.split("->").map((part) => part.trim().replace(/\s+/g, " ")).join(" -> ").replace(/\s+/g, " ");
268004
268101
  }
268005
- function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, authorizeBySource) {
268102
+ function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, gatesBySource) {
268006
268103
  const registry = modelDef.givens ?? {};
268007
268104
  const surfaced = new Set(surfacedGivenNames);
268008
268105
  const givens = new Map;
@@ -268072,7 +268169,7 @@ function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, author
268072
268169
  viewAnnotations,
268073
268170
  sourceFields,
268074
268171
  drills,
268075
- suggestGivens: suggestGivenLookup(modelDef, (source) => authorizeBySource?.get(source), surfaced)
268172
+ suggestGivens: suggestGivenLookup(modelDef, (source) => gatesBySource?.get(source), surfaced)
268076
268173
  };
268077
268174
  }
268078
268175
  function readTileLayout(annotations) {
@@ -270706,20 +270803,33 @@ function recordAuthorizeBypass(entryPoint) {
270706
270803
  }
270707
270804
  function recordRowLevelGateDecision(decision) {
270708
270805
  rowLevelDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_total", {
270709
- description: "How a row-level `#(authorize)` gate resolved a request. Label: decision ('denied_by_gate'|'empty_after_filter'). 'denied_by_gate' is the fail-closed refusal when the gate could not be applied; 'empty_after_filter' is a successful response with zero rows after the filter matched none, which is NOT an error."
270806
+ description: "How a row-level `#(access_filter)` gate resolved a request. Label: decision ('denied_by_gate'|'empty_after_filter'). 'denied_by_gate' is the fail-closed refusal when the gate could not be applied; 'empty_after_filter' is a successful response with zero rows after the filter matched none, which is NOT an error."
270710
270807
  });
270711
270808
  rowLevelDecisionCounter.add(1, { decision });
270712
270809
  }
270713
270810
  function recordRowLevelGateRejected(cause) {
270714
270811
  rowLevelRejectionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_rejected_total", {
270715
- description: "Row-level `#(authorize)` gates that were refused, warned about, or could not be resolved. Label: cause (" + ROW_LEVEL_GATE_REJECTION_CAUSES.map((c) => `'${c}'`).join("|") + "). Only 'legacy_string_gate' fails the whole model load. 'entry_point_unexpressible', 'source_line_gate_no_given_reference' and 'source_line_gate_negated_membership' warn at load and leave the model servable. 'unreachable_given', 'given_usage_unresolvable' and 'unclassifiable_condition' are request-time only — that entry point denies every request. Alert on any nonzero value since the last publish; the request-time causes also make this a rate signal, so see the doc above before writing an alert."
270812
+ description: "Row-level `#(access_filter)` gates that were refused, warned about, or could not be resolved. Label: cause (" + ROW_LEVEL_GATE_REJECTION_CAUSES.map((c) => `'${c}'`).join("|") + "). Only 'legacy_string_gate' fails the whole model load. 'entry_point_unexpressible', 'source_line_gate_no_given_reference' and 'source_line_gate_negated_membership' warn at load and leave the model servable. 'unreachable_given', 'given_usage_unresolvable' and 'unclassifiable_condition' are request-time only — that entry point denies every request. Alert on any nonzero value since the last publish; the request-time causes also make this a rate signal, so see the doc above before writing an alert."
270716
270813
  });
270717
270814
  rowLevelRejectionCounter.add(1, { cause });
270718
270815
  }
270719
- var guardRejectionCounter = null, bypassCounter = null, rowLevelDecisionCounter = null, rowLevelRejectionCounter = null;
270816
+ function recordAuthorizeAdmitAllGate(route) {
270817
+ admitAllCounter ??= publisherMeter().createCounter("publisher_authorize_admit_all_total", {
270818
+ description: "Sources declaring their OWN unconditional admit-all gate (`#(authorize) true`), counted once each at package load. Label: route (" + CANONICAL_AUTHORIZE_ROUTES.map((r) => `'${r}'`).join("|") + "). Not an error — `true` is the only spelling for an extension that deliberately re-opens a gated base — but it is the one declaration that turns a gate off, so a jump since the last publish is worth a look."
270819
+ });
270820
+ admitAllCounter.add(1, { route });
270821
+ }
270822
+ function recordLockDecision(decision) {
270823
+ lockDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_lock_total", {
270824
+ description: "How a `#(authorize)` lock resolved a request. Label: decision ('admitted'|'denied_by_lock'|'denied_unresolvable'). Both denials are a 403; 'denied_by_lock' is the gate refusing a caller it does not admit (routine), 'denied_unresolvable' is the fail-closed refusal when the gate could not be decided at all."
270825
+ });
270826
+ lockDecisionCounter.add(1, { decision });
270827
+ }
270828
+ var guardRejectionCounter = null, bypassCounter = null, rowLevelDecisionCounter = null, rowLevelRejectionCounter = null, admitAllCounter = null, lockDecisionCounter = null;
270720
270829
  var init_authorize_metrics = __esm(() => {
270721
270830
  init_telemetry();
270722
270831
  init_authorize();
270832
+ init_authorize_routes();
270723
270833
  });
270724
270834
 
270725
270835
  // src/materialization_metrics.ts
@@ -271918,6 +272028,317 @@ function filterFreshManifest(entries, now) {
271918
272028
  return { manifest, nextStaleSince };
271919
272029
  }
271920
272030
 
272031
+ // src/service/authorize_grammar.ts
272032
+ import { routeOf as routeOf2 } from "@malloydata/malloy";
272033
+ function noteRoute2(text) {
272034
+ return routeOf2({ value: text.trimStart() });
272035
+ }
272036
+ function splitFieldPathSegments(fieldPath) {
272037
+ const segments = [];
272038
+ let i = 0;
272039
+ while (i < fieldPath.length) {
272040
+ if (fieldPath[i] === "`") {
272041
+ const end = fieldPath.indexOf("`", i + 1);
272042
+ segments.push(fieldPath.slice(i + 1, end));
272043
+ i = end + 1;
272044
+ } else {
272045
+ const dot = fieldPath.indexOf(".", i);
272046
+ const end = dot === -1 ? fieldPath.length : dot;
272047
+ segments.push(fieldPath.slice(i, end));
272048
+ i = end;
272049
+ }
272050
+ if (fieldPath[i] === ".")
272051
+ i++;
272052
+ }
272053
+ return segments;
272054
+ }
272055
+ function rejectionMessage(sourceName, body, detail) {
272056
+ return `Source "${sourceName}" declares an authorize gate \`${body}\`: ${detail} ` + GRAMMAR_SUMMARY;
272057
+ }
272058
+ function reject(sourceName, body, cause, detail) {
272059
+ throw new AuthorizeGrammarError(cause, rejectionMessage(sourceName, body, detail));
272060
+ }
272061
+ function rejectCoherence(sourceName, cause, detail) {
272062
+ throw new AuthorizeGrammarError(cause, `Source "${sourceName}" declares authorize notes whose terms ` + `conflict: ${detail} ${GRAMMAR_SUMMARY}`);
272063
+ }
272064
+ function splitTerms(body) {
272065
+ const terms = [];
272066
+ const compoundTokens = [];
272067
+ let depth = 0;
272068
+ let quote;
272069
+ let start = 0;
272070
+ let i = 0;
272071
+ while (i < body.length) {
272072
+ const ch = body[i];
272073
+ if (quote) {
272074
+ if (ch === "\\" && quote !== "`") {
272075
+ i += 2;
272076
+ continue;
272077
+ }
272078
+ if (ch === quote)
272079
+ quote = undefined;
272080
+ i++;
272081
+ continue;
272082
+ }
272083
+ if (isQuote(ch)) {
272084
+ quote = ch;
272085
+ i++;
272086
+ continue;
272087
+ }
272088
+ if (ch === "(") {
272089
+ depth++;
272090
+ i++;
272091
+ continue;
272092
+ }
272093
+ if (ch === ")") {
272094
+ depth = Math.max(0, depth - 1);
272095
+ i++;
272096
+ continue;
272097
+ }
272098
+ if (depth === 0) {
272099
+ const wordMatch = /^(and|or|not)\b/i.exec(body.slice(i));
272100
+ if (wordMatch && (i === 0 || !/[A-Za-z0-9_]/.test(body[i - 1]))) {
272101
+ const word = wordMatch[1].toLowerCase();
272102
+ if (word === "and") {
272103
+ terms.push(body.slice(start, i));
272104
+ i += wordMatch[0].length;
272105
+ start = i;
272106
+ continue;
272107
+ }
272108
+ compoundTokens.push(word);
272109
+ }
272110
+ }
272111
+ i++;
272112
+ }
272113
+ terms.push(body.slice(start));
272114
+ return { terms, compoundTokens };
272115
+ }
272116
+ function isQuote(ch) {
272117
+ return ch === "'" || ch === '"' || ch === "`";
272118
+ }
272119
+ function findOperatorOutsideQuotes(s, re) {
272120
+ let quote;
272121
+ for (let i = 0;i < s.length; i++) {
272122
+ const ch = s[i];
272123
+ if (quote) {
272124
+ if (ch === "\\" && quote !== "`") {
272125
+ i++;
272126
+ continue;
272127
+ }
272128
+ if (ch === quote)
272129
+ quote = undefined;
272130
+ continue;
272131
+ }
272132
+ if (isQuote(ch)) {
272133
+ quote = ch;
272134
+ continue;
272135
+ }
272136
+ re.lastIndex = i;
272137
+ const m = re.exec(s);
272138
+ if (m && m.index === i)
272139
+ return i;
272140
+ }
272141
+ return -1;
272142
+ }
272143
+ function parseTerm(sourceName, body, term, givenDeclaredTypes) {
272144
+ const trimmed2 = term.trim();
272145
+ if (findOperatorOutsideQuotes(trimmed2, NEGATED_OPERATOR_RE) !== -1) {
272146
+ reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
272147
+ }
272148
+ if (findOperatorOutsideQuotes(trimmed2, COMPARISON_OPERATOR_RE) !== -1) {
272149
+ reject(sourceName, body, "comparison_operator", "only `=` and `in` are allowed, not `<`/`>`/`<=`/`>=`.");
272150
+ }
272151
+ const inIdx = findOperatorOutsideQuotes(trimmed2, IN_OPERATOR_RE);
272152
+ const eqIdx = findOperatorOutsideQuotes(trimmed2, EQ_OPERATOR_RE);
272153
+ let operator;
272154
+ let opIdx;
272155
+ let opLen;
272156
+ if (inIdx !== -1 && (eqIdx === -1 || inIdx < eqIdx)) {
272157
+ operator = "in";
272158
+ opIdx = inIdx;
272159
+ opLen = 2;
272160
+ } else if (eqIdx !== -1) {
272161
+ operator = "=";
272162
+ opIdx = eqIdx;
272163
+ opLen = 1;
272164
+ } else {
272165
+ reject(sourceName, body, "malformed_body", "no `=` or `in` operator was found.");
272166
+ }
272167
+ let left = trimmed2.slice(0, opIdx).trim();
272168
+ let right = trimmed2.slice(opIdx + opLen).trim();
272169
+ if (GIVEN_REF_RE.test(left) && STRING_LITERAL_RE.test(right)) {
272170
+ if (operator === "in") {
272171
+ reject(sourceName, body, "reversed_in_operands", `\`${left} in ${right}\` reads $GIVEN in 'literal', which Malloy ` + `rejects as array-in-string membership. Write the literal ` + `first: \`${right} in ${left}\`.`);
272172
+ }
272173
+ [left, right] = [right, left];
272174
+ }
272175
+ const leftIsFieldPath = FIELD_PATH_RE.test(left);
272176
+ const leftIsLiteral = STRING_LITERAL_RE.test(left);
272177
+ if (!leftIsFieldPath && !leftIsLiteral) {
272178
+ reject(sourceName, body, "left_not_field_path", `\`${left}\` is neither a field path nor a string literal — one ` + "side must be a single column, a dotted join path, or a quoted " + "literal, and the other a `$GIVEN`.");
272179
+ }
272180
+ const givenMatch = GIVEN_REF_RE.exec(right);
272181
+ if (!givenMatch) {
272182
+ reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — a term compares against ` + "`$NAME`.");
272183
+ }
272184
+ const given = givenMatch[1];
272185
+ const declaredType = givenDeclaredTypes.get(given);
272186
+ if (declaredType !== undefined) {
272187
+ const isListType = declaredType === "array" || declaredType.endsWith("[]");
272188
+ if (operator === "=" && isListType) {
272189
+ reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a list-typed given — use \`in\`, not \`=\`.`);
272190
+ }
272191
+ if (operator === "in" && !isListType) {
272192
+ reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a scalar given — use \`=\`, not \`in\`.`);
272193
+ }
272194
+ }
272195
+ if (leftIsLiteral) {
272196
+ return { scope: "source_level", literal: left, given };
272197
+ }
272198
+ return {
272199
+ scope: "row_level",
272200
+ fieldPath: left,
272201
+ fieldPathSegments: splitFieldPathSegments(left),
272202
+ given,
272203
+ operator
272204
+ };
272205
+ }
272206
+ function assertAuthorizeGrammarTermsCoherent(sourceName, terms) {
272207
+ if (terms.length > 1 && terms.some(({ term }) => term.scope === "deny_all")) {
272208
+ rejectCoherence(sourceName, "deny_all_with_sibling", "an unconditional `false` deny-all cannot be combined with any " + "other authorize note — a deny-all admits nothing, so a " + "sibling note can never change what is served and its " + "presence is very likely a mistake.");
272209
+ }
272210
+ const byRoute = new Map;
272211
+ for (const { term, route } of terms) {
272212
+ if (term.scope === "deny_all")
272213
+ continue;
272214
+ const list = byRoute.get(route);
272215
+ if (list)
272216
+ list.push(term);
272217
+ else
272218
+ byRoute.set(route, [term]);
272219
+ }
272220
+ for (const allRouteTerms of byRoute.values()) {
272221
+ if (allRouteTerms.length > 1 && allRouteTerms.some((t) => t.scope === "admit_all")) {
272222
+ rejectCoherence(sourceName, "admit_all_with_sibling", "an unconditional `true` admit-all cannot be combined with " + "another `#(authorize)` note — the lock's notes AND " + "together, so `true and x` reduces to `x` and the admit-all " + "is dead text. To unlock a base while its rows stay " + "filtered, pair the `true` with `#(access_filter)` instead.");
272223
+ }
272224
+ const routeTerms = allRouteTerms.filter((t) => t.scope !== "admit_all" && t.scope !== "deny_all");
272225
+ const seenGivens = new Set;
272226
+ const seenFieldPaths = new Set;
272227
+ for (const t of routeTerms) {
272228
+ if (seenGivens.has(t.given)) {
272229
+ rejectCoherence(sourceName, "duplicate_given", `\`$${t.given}\` is used by more than one term — a given ` + "can back at most one term per route.");
272230
+ }
272231
+ seenGivens.add(t.given);
272232
+ if (t.scope === "row_level") {
272233
+ const canonicalPath = JSON.stringify(t.fieldPathSegments);
272234
+ if (seenFieldPaths.has(canonicalPath)) {
272235
+ rejectCoherence(sourceName, "duplicate_field_path", `\`${t.fieldPath}\` is used by more than one term.`);
272236
+ }
272237
+ seenFieldPaths.add(canonicalPath);
272238
+ }
272239
+ }
272240
+ }
272241
+ }
272242
+ function parseAuthorizeGrammarBody(sourceName, body, givenDeclaredTypes, route) {
272243
+ const trimmedBody = body.trim();
272244
+ if (trimmedBody.length === 0) {
272245
+ reject(sourceName, trimmedBody, "empty_body", "the expression body is empty.");
272246
+ }
272247
+ const sentinel = trimmedBody.toLowerCase();
272248
+ if (sentinel === "false" || sentinel === "true") {
272249
+ if (route !== AUTHORIZE_ROUTE) {
272250
+ reject(sourceName, trimmedBody, "sentinel_in_access_filter", `a bare \`${sentinel}\` is not allowed in \`#(access_filter)\` — ` + (sentinel === "false" ? "write `#(authorize) false`, which refuses the caller " + "rather than serving them zero rows." : "it filters nothing; delete the annotation, or write " + "`#(authorize) true` to re-open a locked base."));
272251
+ }
272252
+ return [{ scope: sentinel === "false" ? "deny_all" : "admit_all" }];
272253
+ }
272254
+ const { terms: rawTerms, compoundTokens } = splitTerms(trimmedBody);
272255
+ if (compoundTokens.length > 0) {
272256
+ reject(sourceName, trimmedBody, "compound_boolean", `a compound boolean (\`${compoundTokens[0]}\`) is not allowed — ` + "join terms with `and` only.");
272257
+ }
272258
+ const parsed = rawTerms.map((term) => parseTerm(sourceName, trimmedBody, term, givenDeclaredTypes));
272259
+ const ADMITTED_SCOPE = {
272260
+ [AUTHORIZE_ROUTE]: "source_level",
272261
+ [ACCESS_FILTER_ROUTE]: "row_level"
272262
+ };
272263
+ const admitted = ADMITTED_SCOPE[route];
272264
+ const offending = parsed.find((t) => t.scope !== admitted);
272265
+ if (offending) {
272266
+ const inLock = route === AUTHORIZE_ROUTE;
272267
+ reject(sourceName, trimmedBody, inLock ? "row_level_term_in_authorize" : "source_level_term_in_access_filter", inLock ? "a row-level term (field on the left) is not allowed in " + "`#(authorize)`, which decides whether the caller may reach " + "the source at all — move the term to `#(access_filter)`." : "a source-level term (literal on the left) is not allowed in " + "`#(access_filter)`, which decides which rows a caller may " + "see — move the term to `#(authorize)`, which refuses a " + "caller it does not admit rather than serving them zero rows.");
272268
+ }
272269
+ assertAuthorizeGrammarTermsCoherent(sourceName, parsed.map((term) => ({ term, route })));
272270
+ return parsed;
272271
+ }
272272
+ function containsRetiredRouteTag(texts) {
272273
+ for (const text of texts) {
272274
+ const route = noteRoute2(text);
272275
+ if (route !== undefined && RETIRED_ROUTES.includes(route)) {
272276
+ return route;
272277
+ }
272278
+ }
272279
+ return;
272280
+ }
272281
+ function reachesRetiredRouteTagBelow(node, seen = new WeakSet, depth = 0) {
272282
+ if (depth > MAX_RETIRED_ROUTE_WALK_DEPTH) {
272283
+ throw new Error("retired-route IR walk exceeded max depth");
272284
+ }
272285
+ if (node === null || typeof node !== "object")
272286
+ return;
272287
+ if (seen.has(node))
272288
+ return;
272289
+ seen.add(node);
272290
+ if (Array.isArray(node)) {
272291
+ for (const item of node) {
272292
+ const found = reachesRetiredRouteTagBelow(item, seen, depth + 1);
272293
+ if (found)
272294
+ return found;
272295
+ }
272296
+ return;
272297
+ }
272298
+ const record = node;
272299
+ if (depth > 0) {
272300
+ for (const key of ["blockNotes", "notes"]) {
272301
+ const arr = record[key];
272302
+ if (!Array.isArray(arr))
272303
+ continue;
272304
+ const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
272305
+ const found = containsRetiredRouteTag(texts);
272306
+ if (found)
272307
+ return found;
272308
+ }
272309
+ }
272310
+ for (const [key, value] of Object.entries(record)) {
272311
+ if (depth === 0 && key === "annotations")
272312
+ continue;
272313
+ const found = reachesRetiredRouteTagBelow(value, seen, depth + 1);
272314
+ if (found)
272315
+ return found;
272316
+ }
272317
+ return;
272318
+ }
272319
+ var AuthorizeGrammarError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", QUOTED_IDENT = "`[^`]+`", SEGMENT, FIELD_PATH_RE, GIVEN_REF_RE, STRING_LITERAL_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE, GRAMMAR_SUMMARY, IN_OPERATOR_RE, EQ_OPERATOR_RE, RETIRED_ROUTES, MAX_RETIRED_ROUTE_WALK_DEPTH = 200;
272320
+ var init_authorize_grammar = __esm(() => {
272321
+ init_errors();
272322
+ init_authorize_routes();
272323
+ AuthorizeGrammarError = class AuthorizeGrammarError extends ModelCompilationError {
272324
+ rejectionCause;
272325
+ constructor(rejectionCause, message) {
272326
+ super({ message });
272327
+ this.rejectionCause = rejectionCause;
272328
+ }
272329
+ };
272330
+ SEGMENT = `(?:${IDENT}|${QUOTED_IDENT})`;
272331
+ FIELD_PATH_RE = new RegExp(`^${SEGMENT}(?:\\.${SEGMENT})*$`);
272332
+ GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
272333
+ STRING_LITERAL_RE = /^(?:'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")$/;
272334
+ NEGATED_OPERATOR_RE = /!=/gu;
272335
+ COMPARISON_OPERATOR_RE = /(?:>=|<=|>|<)/gu;
272336
+ GRAMMAR_SUMMARY = "An authorize gate only accepts one or more `and`-joined terms, each " + "either `<column> <op> $GIVEN` (a single field or a dotted join " + "path on the left) or `'<literal>' in $GIVEN` (a literal on the " + "left), where <op> is fixed by the given's own arity: `in` for a " + "list-typed given, `=` for a scalar one.";
272337
+ IN_OPERATOR_RE = /\bin\b/giu;
272338
+ EQ_OPERATOR_RE = /=/gu;
272339
+ RETIRED_ROUTES = ["partition"];
272340
+ });
272341
+
271921
272342
  // src/service/gate_dimension.ts
271922
272343
  import {
271923
272344
  isJoined,
@@ -271982,11 +272403,12 @@ function containsNegatedMembership(node, depth = 0) {
271982
272403
  return containsNegatedMembership(n.e, depth + 1);
271983
272404
  return false;
271984
272405
  }
271985
- function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, conditionExpr, modelDef, onWarning) {
272406
+ function validateSourceLineGateGivenUsage(sourceName, route, struct, refSummary, conditionExpr, modelDef, onWarning) {
272407
+ const gate = `#(${route}) gate`;
271986
272408
  const expansion = expandRefSummaryGivenIds(struct, refSummary);
271987
272409
  if (!expansion.ok) {
271988
272410
  throw new ModelCompilationError({
271989
- message: `#(authorize) on source "${sourceName}" references ` + `"${expansion.unresolvedPath}", which could not be resolved to a ` + `field this model can reach. An unresolvable reference is refused, ` + `not treated as referencing no given`
272411
+ message: `The ${gate} on source "${sourceName}" references ` + `"${expansion.unresolvedPath}", which could not be resolved to a ` + `field this model can reach. An unresolvable reference is refused, ` + `not treated as referencing no given`
271990
272412
  });
271991
272413
  }
271992
272414
  const givenIds = expansion.givenIds;
@@ -271994,15 +272416,15 @@ function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, condit
271994
272416
  const given = modelDef.givens?.[id];
271995
272417
  if (given && (given.default !== undefined || given.defaultText !== undefined)) {
271996
272418
  throw new ModelCompilationError({
271997
- message: `#(authorize) on source "${sourceName}" references ` + `\`$${given.name}\`, which is declared with a default. A ` + `caller who supplies no value for \`$${given.name}\` gets ` + `that default, which can admit rows the gate was meant to ` + `exclude. Declare \`$${given.name}\` with no default`
272419
+ message: `The ${gate} on source "${sourceName}" references ` + `\`$${given.name}\`, which is declared with a default. A ` + `caller who supplies no value for \`$${given.name}\` gets ` + `that default, which can admit what the gate was meant to ` + `exclude. Declare \`$${given.name}\` with no default`
271998
272420
  });
271999
272421
  }
272000
272422
  }
272001
272423
  if (givenIds.size === 0) {
272002
- onWarning?.("source_line_gate_no_given_reference", `#(authorize) on source "${sourceName}" references no given; it is ` + `a fixed predicate, not an access rule keyed on the caller`);
272424
+ onWarning?.("source_line_gate_no_given_reference", `The ${gate} on source "${sourceName}" references no given; it is ` + `a fixed predicate, not an access rule keyed on the caller`);
272003
272425
  }
272004
272426
  if (containsNegatedMembership(conditionExpr)) {
272005
- onWarning?.("source_line_gate_negated_membership", `#(authorize) on source "${sourceName}" negates a membership test; ` + `an empty given then matches every row instead of none`);
272427
+ onWarning?.("source_line_gate_negated_membership", `The ${gate} on source "${sourceName}" negates a membership test; ` + `an empty given then matches everything instead of nothing`);
272006
272428
  }
272007
272429
  }
272008
272430
  var init_gate_dimension = __esm(() => {
@@ -272030,10 +272452,10 @@ function resolveDeclaredSource(struct, modelDef) {
272030
272452
  }
272031
272453
  return sawBrokenEntry ? { kind: "unresolvable" } : { kind: "none" };
272032
272454
  }
272033
- function ancestorGateExprs(struct, modelDef, seen = new Set) {
272455
+ function ancestorGateExprs(struct, modelDef, route, seen = new Set) {
272034
272456
  let inherited = struct.annotations?.inherits;
272035
272457
  for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
272036
- const exprs2 = collectAuthorizeExprs(ownLevelNoteTexts(inherited));
272458
+ const exprs2 = collectAuthorizeExprsForRoute(ownLevelNoteTexts(inherited), route);
272037
272459
  if (exprs2.length > 0)
272038
272460
  return exprs2;
272039
272461
  inherited = inherited.inherits;
@@ -272041,15 +272463,17 @@ function ancestorGateExprs(struct, modelDef, seen = new Set) {
272041
272463
  if (inherited)
272042
272464
  return ["false"];
272043
272465
  seen.add(struct);
272044
- if (seen.size > ANCESTOR_WALK_MAX_DEPTH)
272466
+ if (seen.size > ANCESTOR_WALK_MAX_DEPTH) {
272045
272467
  return ["false"];
272468
+ }
272046
272469
  const declared = resolveDeclaredSource(struct, modelDef);
272047
- if (declared.kind === "unresolvable")
272470
+ if (declared.kind === "unresolvable") {
272048
272471
  return ["false"];
272472
+ }
272049
272473
  if (declared.kind === "none" || seen.has(declared.source))
272050
272474
  return [];
272051
- const exprs = collectAuthorizeExprs(ownLevelNoteTexts(declared.source.annotations));
272052
- return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, seen);
272475
+ const exprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(declared.source.annotations), route);
272476
+ return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, route, seen);
272053
272477
  }
272054
272478
  function resolveQuerySourceBase(struct, modelDef) {
272055
272479
  const duck = struct;
@@ -272063,8 +272487,8 @@ function resolveCompositeResolvedBase(struct) {
272063
272487
  const duck = struct;
272064
272488
  return duck.type === "query_source" ? duck.query?.compositeResolvedSourceDef : undefined;
272065
272489
  }
272066
- function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
272067
- const direct = ancestorGateExprs(struct, modelDef, new Set(seen));
272490
+ function effectiveAncestorGateExprs(struct, modelDef, route, seen = new Set) {
272491
+ const direct = ancestorGateExprs(struct, modelDef, route, new Set(seen));
272068
272492
  if (direct.length > 0)
272069
272493
  return [direct];
272070
272494
  if (seen.has(struct))
@@ -272074,18 +272498,19 @@ function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
272074
272498
  const base = resolveQuerySourceBase(struct, modelDef);
272075
272499
  if (!base) {
272076
272500
  const duck = struct;
272077
- if (duck.type === "query_source")
272501
+ if (duck.type === "query_source") {
272078
272502
  groups.push(["false"]);
272503
+ }
272079
272504
  } else if (!seen.has(base)) {
272080
- const ownExprs = collectAuthorizeExprs(ownLevelNoteTexts(base.annotations));
272081
- groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, seen));
272505
+ const ownExprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(base.annotations), route);
272506
+ groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, route, seen));
272082
272507
  }
272083
272508
  const composite = resolveCompositeResolvedBase(struct);
272084
272509
  if (composite && !seen.has(composite)) {
272085
272510
  const parentOwnNotes = base ? ownLevelNotes(base.annotations) : [];
272086
272511
  const compositeOwnNotes = ownLevelNotes(composite.annotations).filter((note) => !parentOwnNotes.includes(note));
272087
- const compositeOwn = collectAuthorizeExprs(compositeOwnNotes.map((note) => note.text));
272088
- groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, seen));
272512
+ const compositeOwn = collectAuthorizeExprsForRoute(compositeOwnNotes.map((note) => note.text), route);
272513
+ groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, route, seen));
272089
272514
  }
272090
272515
  return groups;
272091
272516
  }
@@ -272114,120 +272539,9 @@ var init_gate_registry_walk = __esm(() => {
272114
272539
  init_authorize();
272115
272540
  });
272116
272541
 
272117
- // src/service/partition_annotation.ts
272118
- import { payloadOf as payloadOf2, routeOf as routeOf2 } from "@malloydata/malloy";
272119
- function noteRoute2(text) {
272120
- return routeOf2({ value: text.trimStart() });
272121
- }
272122
- function containsPartitionAnnotationTag(texts) {
272123
- return texts.some((text) => noteRoute2(text) === PARTITION_ROUTE);
272124
- }
272125
- function reachesPartitionTagBelow(node, seen = new WeakSet, depth = 0) {
272126
- if (depth > MAX_PARTITION_IR_WALK_DEPTH) {
272127
- throw new Error("partition-marker IR walk exceeded max depth");
272128
- }
272129
- if (node === null || typeof node !== "object")
272130
- return false;
272131
- if (seen.has(node))
272132
- return false;
272133
- seen.add(node);
272134
- if (Array.isArray(node)) {
272135
- return node.some((item) => reachesPartitionTagBelow(item, seen, depth + 1));
272136
- }
272137
- const record = node;
272138
- if (depth > 0 && record.join !== undefined)
272139
- return false;
272140
- if (depth > 0) {
272141
- for (const key of ["blockNotes", "notes"]) {
272142
- const arr = record[key];
272143
- if (!Array.isArray(arr))
272144
- continue;
272145
- const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
272146
- if (containsPartitionAnnotationTag(texts))
272147
- return true;
272148
- }
272149
- }
272150
- return Object.entries(record).some(([key, value]) => depth === 0 && key === "annotations" ? false : reachesPartitionTagBelow(value, seen, depth + 1));
272151
- }
272152
- function notePayload2(text) {
272153
- return payloadOf2({ value: text.trimStart() }) ?? "";
272154
- }
272155
- function rejectionMessage(sourceName, body, detail) {
272156
- return `Source "${sourceName}" declares \`#(partition) ${body}\`: ${detail} ` + `#(partition) only accepts \`<column> = $GIVEN\`, where <column> is a ` + `single field or a dotted join path.`;
272157
- }
272158
- function reject(sourceName, body, cause, detail) {
272159
- throw new PartitionAnnotationError(cause, rejectionMessage(sourceName, body, detail));
272160
- }
272161
- function parsePartitionAnnotation(sourceName, annotationText) {
272162
- if (noteRoute2(annotationText) !== PARTITION_ROUTE)
272163
- return null;
272164
- const body = notePayload2(annotationText).trim();
272165
- if (body.length === 0) {
272166
- reject(sourceName, body, "empty_body", "the expression body is empty.");
272167
- }
272168
- if (COMPOUND_BOOLEAN_RE.test(body)) {
272169
- reject(sourceName, body, "compound_boolean", "a compound boolean (`and`/`or`/`not`) is not allowed — declare one " + "`#(partition)` marker per column.");
272170
- }
272171
- if (IN_OPERATOR_RE.test(body)) {
272172
- reject(sourceName, body, "in_operator", "the `in` operator is not allowed.");
272173
- }
272174
- if (NEGATED_OPERATOR_RE.test(body)) {
272175
- reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
272176
- }
272177
- if (COMPARISON_OPERATOR_RE.test(body)) {
272178
- reject(sourceName, body, "comparison_operator", "only `=` is allowed, not `<`/`>`/`<=`/`>=`.");
272179
- }
272180
- const eq = body.indexOf("=");
272181
- if (eq === -1) {
272182
- reject(sourceName, body, "malformed_body", "no `=` was found.");
272183
- }
272184
- const left = body.slice(0, eq).trim();
272185
- const right = body.slice(eq + 1).trim();
272186
- if (!FIELD_PATH_RE.test(left)) {
272187
- reject(sourceName, body, "left_not_field_path", `\`${left}\` is not a field path — the left side must be a single ` + "column or a dotted join path, not an expression.");
272188
- }
272189
- const givenMatch = GIVEN_REF_RE.exec(right);
272190
- if (!givenMatch) {
272191
- reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — the right side must be ` + "`$NAME`.");
272192
- }
272193
- return { column: left, given: givenMatch[1] };
272194
- }
272195
- function collectPartitionPairs(sourceName, annotationTexts2) {
272196
- const pairs = [];
272197
- const seenGivens = new Map;
272198
- for (const text of annotationTexts2) {
272199
- const pair = parsePartitionAnnotation(sourceName, text);
272200
- if (pair === null)
272201
- continue;
272202
- const priorColumn = seenGivens.get(pair.given);
272203
- if (priorColumn !== undefined) {
272204
- throw new PartitionAnnotationError("duplicate_given", `Source "${sourceName}" declares \`#(partition)\` on both ` + `\`${priorColumn}\` and \`${pair.column}\` for the same given ` + `\`$${pair.given}\` — a given can back at most one partition ` + `column per source.`);
272205
- }
272206
- seenGivens.set(pair.given, pair.column);
272207
- pairs.push(pair);
272208
- }
272209
- return pairs;
272210
- }
272211
- var PARTITION_ROUTE = "partition", MAX_PARTITION_IR_WALK_DEPTH = 200, PartitionAnnotationError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", FIELD_PATH_RE, GIVEN_REF_RE, COMPOUND_BOOLEAN_RE, IN_OPERATOR_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE;
272212
- var init_partition_annotation = __esm(() => {
272213
- init_errors();
272214
- PartitionAnnotationError = class PartitionAnnotationError extends ModelCompilationError {
272215
- rejectionCause;
272216
- constructor(rejectionCause, message) {
272217
- super({ message });
272218
- this.rejectionCause = rejectionCause;
272219
- }
272220
- };
272221
- FIELD_PATH_RE = new RegExp(`^${IDENT}(?:\\.${IDENT})*$`);
272222
- GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
272223
- COMPOUND_BOOLEAN_RE = /\b(and|or|not)\b/i;
272224
- IN_OPERATOR_RE = /\bin\b/i;
272225
- NEGATED_OPERATOR_RE = /!=/;
272226
- COMPARISON_OPERATOR_RE = /(>=|<=|>|<)/;
272227
- });
272228
-
272229
272542
  // src/service/gate_classification.ts
272230
272543
  import {
272544
+ isJoined as isJoined2,
272231
272545
  isSourceDef as isSourceDef5
272232
272546
  } from "@malloydata/malloy";
272233
272547
  function createGateClassificationDeps(givens, modelPath) {
@@ -272237,30 +272551,41 @@ function createGateClassificationDeps(givens, modelPath) {
272237
272551
  modelPath
272238
272552
  };
272239
272553
  }
272240
- function gateExprsForOwnAnnotations(struct, modelDef, excludeNotes = []) {
272554
+ function gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes = []) {
272241
272555
  const ownNotes = ownLevelNotes(struct.annotations).filter((note) => !excludeNotes.includes(note));
272242
272556
  try {
272243
- const own = collectAuthorizeExprs(ownNotes.map((note) => note.text));
272557
+ const own = collectAuthorizeExprsForRoute(ownNotes.map((note) => note.text), route);
272244
272558
  if (own.length > 0) {
272245
272559
  return { exprs: own, fromAncestor: false };
272246
272560
  }
272247
- const ancestor = ancestorGateExprs(struct, modelDef);
272561
+ const ancestor = ancestorGateExprs(struct, modelDef, route);
272248
272562
  return { exprs: ancestor, fromAncestor: ancestor.length > 0 };
272249
272563
  } catch {
272250
- return { exprs: ["false"], fromAncestor: false };
272564
+ return {
272565
+ exprs: ["false"],
272566
+ fromAncestor: false
272567
+ };
272251
272568
  }
272252
272569
  }
272253
272570
  function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate = false, entryPointStruct = struct, excludeNotes = []) {
272571
+ const results = [];
272572
+ for (const route of CANONICAL_AUTHORIZE_ROUTES) {
272573
+ results.push(...collectEntryPointGatesForRoute(struct, modelDef, route, new Set(seen), treatAsOwnGate, entryPointStruct, excludeNotes));
272574
+ }
272575
+ return results;
272576
+ }
272577
+ function collectEntryPointGatesForRoute(struct, modelDef, route, seen, treatAsOwnGate, entryPointStruct, excludeNotes) {
272254
272578
  if (!struct || !modelDef || seen.has(struct))
272255
272579
  return [];
272256
272580
  seen.add(struct);
272257
272581
  const results = [];
272258
272582
  const label = struct.as ?? struct.name;
272259
- const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, modelDef, excludeNotes);
272583
+ const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes);
272260
272584
  if (ownExprs.length > 0) {
272261
272585
  results.push({
272262
272586
  label,
272263
272587
  exprs: ownExprs,
272588
+ route,
272264
272589
  selfContained: fromAncestor || !treatAsOwnGate,
272265
272590
  struct: entryPointStruct
272266
272591
  });
@@ -272269,17 +272594,18 @@ function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate
272269
272594
  if (duck.type === "query_source") {
272270
272595
  const base = resolveQuerySourceBase(struct, modelDef);
272271
272596
  if (base) {
272272
- results.push(...collectEntryPointGates(base, modelDef, seen, false, entryPointStruct));
272597
+ results.push(...collectEntryPointGatesForRoute(base, modelDef, route, seen, false, entryPointStruct, []));
272273
272598
  } else {
272274
272599
  results.push({
272275
272600
  label,
272276
272601
  exprs: ["false"],
272602
+ route,
272277
272603
  selfContained: true
272278
272604
  });
272279
272605
  }
272280
272606
  const resolved = duck.query?.compositeResolvedSourceDef;
272281
272607
  if (resolved) {
272282
- results.push(...collectEntryPointGates(resolved, modelDef, seen, false, entryPointStruct, base ? ownLevelNotes(base.annotations) : []));
272608
+ results.push(...collectEntryPointGatesForRoute(resolved, modelDef, route, seen, false, entryPointStruct, base ? ownLevelNotes(base.annotations) : []));
272283
272609
  }
272284
272610
  }
272285
272611
  return results;
@@ -272296,7 +272622,7 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
272296
272622
  return { shape: "rejected" };
272297
272623
  }
272298
272624
  const filterText = gateFilterText(entry.exprs);
272299
- const cacheKey = `${graftScope.cacheScope}\x00${graftTarget}\x00${filterText}`;
272625
+ const cacheKey = `${graftScope.cacheScope}\x00${entry.route}\x00${graftTarget}\x00${filterText}`;
272300
272626
  let cached2 = deps.gateShapeCache.get(cacheKey);
272301
272627
  if (!cached2) {
272302
272628
  let condition;
@@ -272319,7 +272645,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
272319
272645
  detail: "this entry's lifted condition carries no usable expression"
272320
272646
  };
272321
272647
  } else if (isBareFalseLiteral(condition.e)) {
272322
- classification = { shape: "row_level", givenNames: [] };
272648
+ classification = {
272649
+ shape: "row_level",
272650
+ givenNames: [],
272651
+ givenNamesById: new Map
272652
+ };
272323
272653
  } else {
272324
272654
  const targetStruct = graftScope.modelDef.contents[graftTarget];
272325
272655
  if (!isSourceDef5(targetStruct)) {
@@ -272337,7 +272667,13 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
272337
272667
  detail: `this gate references \`${expansion.unresolvedPath}\`, which could not be resolved on the graft target`
272338
272668
  };
272339
272669
  } else {
272340
- const givenNames = Array.from(expansion.givenIds).map((id) => graftScope.modelDef.givens?.[id]?.name).filter((name) => !!name);
272670
+ const givenNamesById = new Map;
272671
+ for (const id of expansion.givenIds) {
272672
+ const name = graftScope.modelDef.givens?.[id]?.name;
272673
+ if (name)
272674
+ givenNamesById.set(id, name);
272675
+ }
272676
+ const givenNames = Array.from(givenNamesById.values());
272341
272677
  const literalNames = referencedGivenNames(filterText);
272342
272678
  const accountedFor = new Set(givenNames);
272343
272679
  const unaccounted = literalNames.filter((name) => !accountedFor.has(name));
@@ -272348,7 +272684,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
272348
272684
  detail: "this gate references a given id that does not resolve to a name on this model"
272349
272685
  };
272350
272686
  } else {
272351
- classification = { shape: "row_level", givenNames };
272687
+ classification = {
272688
+ shape: "row_level",
272689
+ givenNames,
272690
+ givenNamesById
272691
+ };
272352
272692
  }
272353
272693
  }
272354
272694
  }
@@ -272374,6 +272714,14 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
272374
272714
  if (cached2.classification.shape === "rejected") {
272375
272715
  return { shape: "rejected", cause: cached2.classification.cause };
272376
272716
  }
272717
+ if (entry.route === AUTHORIZE_ROUTE) {
272718
+ return {
272719
+ shape: "lock",
272720
+ condition: cached2.condition,
272721
+ givenNames: cached2.classification.givenNames,
272722
+ givenNamesById: cached2.classification.givenNamesById
272723
+ };
272724
+ }
272377
272725
  return {
272378
272726
  shape: "row_level",
272379
272727
  graftTarget,
@@ -272456,142 +272804,120 @@ function isBareFalseLiteral(expr) {
272456
272804
  function computeGivenDeclaredTypes(givens) {
272457
272805
  return new Map((givens ?? []).filter((g) => g.name != null && g.type != null).map((g) => [g.name, g.type]));
272458
272806
  }
272459
- function resolveEntryPointPartitions(struct, modelDef, seen = new Set) {
272460
- if (!struct || !modelDef || seen.has(struct))
272461
- return [];
272462
- seen.add(struct);
272463
- const label = struct.as ?? struct.name;
272464
- const own = collectPartitionPairs(label, ownLevelNotes(struct.annotations).map((note) => note.text));
272465
- if (own.length > 0)
272466
- return own;
272467
- let inherited = struct.annotations?.inherits;
272468
- for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
272469
- const pairs = collectPartitionPairs(label, ownLevelNoteTexts(inherited));
272470
- if (pairs.length > 0)
272471
- return pairs;
272472
- inherited = inherited.inherits;
272473
- }
272474
- if (inherited)
272475
- throw unresolvableAncestry(label);
272476
- const declared = resolveDeclaredSource(struct, modelDef);
272477
- if (declared.kind === "unresolvable")
272478
- throw unresolvableAncestry(label);
272479
- if (declared.kind === "resolved") {
272480
- const pairs = resolveEntryPointPartitions(declared.source, modelDef, seen);
272481
- if (pairs.length > 0)
272482
- return pairs;
272483
- }
272484
- const queryBase = resolveQuerySourceBase(struct, modelDef);
272485
- if (queryBase)
272486
- return resolveEntryPointPartitions(queryBase, modelDef, seen);
272487
- return [];
272488
- }
272489
- function unresolvableAncestry(label) {
272490
- return new PartitionAnnotationError("ancestry_unresolvable", `Could not resolve whether source "${label}" carries a ` + `\`#(partition)\` marker: the derivation chain it inherits from ` + `could not be read. Denying rather than serving unfiltered.`);
272491
- }
272492
- function assertPartitionAnnotationsValid(modelDef) {
272807
+ function assertAuthorizeGrammarValid(modelDef, authorizeMap, authorizeOwnNotes, givenDeclaredTypes, onAdmitAllGate, attributedAuthorizeOwnNotes) {
272493
272808
  if (!modelDef)
272494
272809
  return;
272495
- for (const [key, obj] of Object.entries(modelDef.contents)) {
272496
- if (!isSourceDef5(obj))
272497
- continue;
272498
- const label = obj.as ?? obj.name ?? key;
272499
- if (obj.type !== "composite") {
272500
- let resolved = [];
272501
- try {
272502
- resolved = resolveEntryPointPartitions(obj, modelDef);
272503
- } catch (err) {
272504
- if (!(err instanceof PartitionAnnotationError) || err.rejectionCause !== "ancestry_unresolvable") {
272505
- throw err;
272810
+ for (const [sourceName, groups] of authorizeMap) {
272811
+ const struct = modelDef.contents[sourceName];
272812
+ const ownGroupTerms = [];
272813
+ const declaredAdmitAllRoutes = [];
272814
+ for (const { route, exprs } of groups) {
272815
+ const isOwn = (authorizeOwnNotes.get(sourceName)?.get(route)?.length ?? 0) > 0;
272816
+ const filteredExprs = exprs.filter((expr) => (isOwn || expr !== "false") && !isLegacyQuotedPayload(expr));
272817
+ const groupTerms = [];
272818
+ for (const expr of filteredExprs) {
272819
+ const terms = parseAuthorizeGrammarBody(sourceName, expr, givenDeclaredTypes, route);
272820
+ for (const term of terms) {
272821
+ groupTerms.push({ term, route });
272822
+ if (term.scope === "admit_all" && (attributedAuthorizeOwnNotes?.get(sourceName)?.get(route)?.length ?? 0) > 0) {
272823
+ declaredAdmitAllRoutes.push(route);
272824
+ }
272825
+ if (term.scope === "row_level" && isSourceDef5(struct)) {
272826
+ assertNoFanoutFieldPath(sourceName, struct, term.fieldPath, term.fieldPathSegments);
272827
+ }
272506
272828
  }
272507
- continue;
272508
272829
  }
272509
- assertNoUnreachableMarker(obj, label, resolved);
272510
- continue;
272830
+ if (isOwn) {
272831
+ ownGroupTerms.push(...groupTerms);
272832
+ } else {
272833
+ assertAuthorizeGrammarTermsCoherent(sourceName, groupTerms);
272834
+ }
272511
272835
  }
272512
- if (resolveEntryPointPartitions(obj, modelDef).length > 0) {
272513
- throw new PartitionAnnotationError("partitioned_composite", `Source "${label}" is a composite source (\`compose(...)\`) and ` + `also declares \`#(partition)\` (on itself or an ancestor it ` + `derives from). A composite run target compiles each query ` + `against one resolved member branch, which a partition filter ` + `cannot attach to — declare \`#(partition)\` on a non-composite ` + `source instead.`);
272836
+ if (ownGroupTerms.length > 0) {
272837
+ assertAuthorizeGrammarTermsCoherent(sourceName, ownGroupTerms);
272514
272838
  }
272515
- const marked = partitionedMemberLabel(obj, modelDef);
272516
- if (marked !== undefined) {
272517
- throw new PartitionAnnotationError("partitioned_composite", `Source "${label}" is a composite source (\`compose(...)\`) with ` + `member "${marked}", which declares \`#(partition)\`. MEASURED: ` + `querying the composite reads EVERY partition of that member — ` + `the graft lands on the composite's own struct, not the ` + `resolved member branch, so no filter reaches the executed ` + `query. Remove the composite, or drop \`#(partition)\` from ` + `"${marked}" and expose it as its own source.`);
272839
+ for (const route of declaredAdmitAllRoutes) {
272840
+ onAdmitAllGate?.(sourceName, route);
272518
272841
  }
272519
272842
  }
272520
272843
  }
272521
- function assertNoUnreachableMarker(struct, label, resolved) {
272522
- if (resolved.length > 0)
272523
- return;
272524
- let reachesBelow;
272525
- try {
272526
- reachesBelow = reachesPartitionTagBelow(struct);
272527
- } catch {
272528
- reachesBelow = true;
272529
- }
272530
- if (!reachesBelow)
272531
- return;
272532
- throw new PartitionAnnotationError("marker_unreachable", `Source "${label}" carries a \`#(partition)\` marker that is not ` + `declared on the source itself — it sits on a field, a view, or an ` + `inline \`compose(...)\` inside it. Nothing would filter this ` + `source: every caller would read every partition. Move the marker ` + `onto the \`source:\` declaration.`);
272533
- }
272534
- function partitionedMemberLabel(composite, modelDef, seen = new Set) {
272535
- if (seen.has(composite))
272536
- return;
272537
- seen.add(composite);
272538
- for (const member of composite.sources ?? []) {
272539
- const label = member.as ?? member.name;
272540
- if (resolveEntryPointPartitions(member, modelDef).length > 0) {
272541
- return label;
272844
+ function fanoutSegment(struct, path9) {
272845
+ let current = struct;
272846
+ for (let i = 0;i < path9.length - 1; i++) {
272847
+ const seg = path9[i];
272848
+ const field = (current.fields ?? []).find((f) => (f.as ?? f.name) === seg);
272849
+ if (!field)
272850
+ return { segment: seg, reason: "unresolved" };
272851
+ if (isJoined2(field) && field.join !== "one") {
272852
+ return { segment: seg, reason: "fanout" };
272542
272853
  }
272543
- if (member.type === "composite") {
272544
- const nested = partitionedMemberLabel(member, modelDef, seen);
272545
- if (nested !== undefined)
272546
- return nested;
272854
+ if (!isSourceDef5(field) && field.type !== "record" && field.type !== "array") {
272855
+ return { segment: seg, reason: "unresolved" };
272547
272856
  }
272857
+ current = field;
272548
272858
  }
272549
272859
  return;
272550
272860
  }
272551
- async function resolvePartitionGraftEntries(struct, originModelDef, graftScope, deps) {
272552
- if (!struct || !originModelDef)
272553
- return [];
272554
- const pairs = resolveEntryPointPartitions(struct, originModelDef);
272555
- if (pairs.length === 0)
272556
- return [];
272557
- const label = struct.as ?? struct.name;
272558
- if (!graftScope) {
272559
- logger.debug("Partition filter has no graft scope to attach to; denying", { modelPath: deps.modelPath, label });
272560
- throw new Error(`partition on "${label}" has no graft scope`);
272561
- }
272562
- const graftTarget = resolveGraftTarget(struct, originModelDef, graftScope.modelDef);
272563
- if (!graftTarget) {
272564
- logger.debug("Partition filter resolved to no graft target; denying", {
272565
- modelPath: deps.modelPath,
272566
- label
272567
- });
272568
- throw new Error(`partition on "${label}" resolved to no graft target`);
272861
+ function assertNoFanoutFieldPath(sourceName, struct, fieldPath, fieldPathSegments) {
272862
+ const finding = fanoutSegment(struct, fieldPathSegments);
272863
+ if (finding === undefined)
272864
+ return;
272865
+ const detail = finding.reason === "fanout" ? `which reaches through \`${finding.segment}\`, a fan-out join ` + `(\`join_many\`/\`join_cross\`, or a repeated record) — the ` + `entry row has many values for what follows, so there is no ` + `single key to compare against a given. Gate on a field reached ` + `only through \`join_one\` joins.` : `whose segment \`${finding.segment}\` does not resolve against ` + `this source — refused rather than assumed safe.`;
272866
+ throw new AuthorizeGrammarError("fanout_path", `Source "${sourceName}" declares an authorize gate on \`${fieldPath}\`, ${detail}`);
272867
+ }
272868
+ function collectRetiredRouteMarkers(modelDef) {
272869
+ const found = [];
272870
+ if (!modelDef)
272871
+ return found;
272872
+ const fileMatch = containsRetiredRouteTag(ownLevelNoteTexts(modelAnnotations(modelDef)));
272873
+ if (fileMatch) {
272874
+ found.push(`the model itself declares a retired \`##(${fileMatch})\` annotation`);
272569
272875
  }
272570
- const entries = [];
272571
- for (const pair of pairs) {
272572
- if (!deps.givenDeclaredTypes.has(pair.given)) {
272573
- logger.warn("Partition references a given off the model surface; denying", { modelPath: deps.modelPath, graftTarget, givenName: pair.given });
272574
- throw new Error(`partition on "${label}" references \`$${pair.given}\`, which is not on this model's given surface`);
272876
+ for (const [key, obj] of Object.entries(modelDef.contents)) {
272877
+ const isQuery = obj.type === "query";
272878
+ if (!isSourceDef5(obj) && !isQuery)
272879
+ continue;
272880
+ const label = obj.as ?? obj.name ?? key;
272881
+ const noun = isQuery ? "query" : "source";
272882
+ const ownMatch = containsRetiredRouteTag(annotationTexts(obj.annotations) ?? []);
272883
+ if (ownMatch) {
272884
+ found.push(`${noun} "${label}" declares a retired \`#(${ownMatch})\` annotation`);
272885
+ continue;
272886
+ }
272887
+ if (isQuery) {
272888
+ continue;
272889
+ }
272890
+ let belowMatch;
272891
+ try {
272892
+ belowMatch = reachesRetiredRouteTagBelow(obj);
272893
+ } catch {
272894
+ belowMatch = RETIRED_ROUTES[0];
272895
+ }
272896
+ if (belowMatch) {
272897
+ found.push(`source "${label}" carries a retired \`#(${belowMatch})\` ` + `annotation somewhere in its IR (not on the source line itself)`);
272575
272898
  }
272576
- const filterText = `${pair.column} = $${pair.given}`;
272577
- const condition = await liftGateCondition(graftTarget, filterText, graftScope.materializer);
272578
- entries.push({
272579
- label,
272580
- graftTarget,
272581
- filterText,
272582
- condition,
272583
- givenNames: [pair.given]
272584
- });
272585
272899
  }
272586
- return entries;
272900
+ return found;
272901
+ }
272902
+ function assertNoRetiredRouteMarkers(found) {
272903
+ if (found.length === 0)
272904
+ return;
272905
+ throw new ModelCompilationError({
272906
+ message: `These sources carry a marker from a retired annotation route:
272907
+ ` + `${found.map((f) => ` - ${f}`).join(`
272908
+ `)}
272909
+ ` + `The route is no longer enforced by anything, so Malloy would ` + `otherwise parse it, load the model clean, and serve every row as ` + `if the marker were never written. Remove the annotation.`
272910
+ });
272587
272911
  }
272588
272912
  var init_gate_classification = __esm(() => {
272913
+ init_errors();
272589
272914
  init_logger();
272590
272915
  init_annotations();
272591
272916
  init_authorize();
272917
+ init_authorize_grammar();
272918
+ init_authorize_routes();
272592
272919
  init_gate_dimension();
272593
272920
  init_gate_registry_walk();
272594
- init_partition_annotation();
272595
272921
  });
272596
272922
 
272597
272923
  // src/service/persist_dynamic_terms.ts
@@ -273061,14 +273387,7 @@ function assertMaterializationEligible(persistSource, annotationFields) {
273061
273387
  recordEligibilityRefused("authorize");
273062
273388
  throw new MaterializationEligibilityError({
273063
273389
  reason: "authorize",
273064
- message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by an #(authorize) gate (its own or a ` + `joined source's). An authorize expression is evaluated per request; ` + `a materialized-once table served frozen carries no gate, so it would ` + `be served to everyone, bypassing authorization. This is refused for ` + `safety. Serve this source live (drop 'storage=').`
273065
- });
273066
- }
273067
- if (referencesPartition(persistSource)) {
273068
- recordEligibilityRefused("partition");
273069
- throw new MaterializationEligibilityError({
273070
- reason: "partition",
273071
- message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it declares a #(partition) marker. A partition ` + `filter binds a given at query time (the same mechanism as ` + `row-level access control), so a materialized-once table served ` + `frozen would leak every partition's rows to every caller. This ` + `is refused for safety. Serve this source live (drop 'storage=').`
273390
+ message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by a gate — #(authorize) or ` + `#(access_filter), its own or a joined source's. Both are evaluated ` + `per request; a materialized-once table served frozen carries no ` + `gate, so it would be served to everyone, bypassing authorization. ` + `This is refused for safety. Serve this source live (drop ` + `'storage=').`
273072
273391
  });
273073
273392
  }
273074
273393
  const partition = resolvePartitionColumns(persistSource, annotationFields);
@@ -273104,15 +273423,6 @@ function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = per
273104
273423
  message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${partition.detail}.`
273105
273424
  });
273106
273425
  }
273107
- if (referencesPartition(persistSource)) {
273108
- recordEligibilityRefused("partition");
273109
- const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
273110
- const gated2 = origin2 === "preaggregate" ? `the source '${sourceName}' rolls up declares` : `it declares`;
273111
- throw new MaterializationEligibilityError({
273112
- reason: "partition",
273113
- message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${gated2} a #(partition) marker. A partition filter binds a ` + `given at query time; this pass cannot prove the persisted ` + `artifact's read path still applies it, so this is refused for ` + `safety. Move the marker to a source that is not materialized, ` + `or stop persisting this one.`
273114
- });
273115
- }
273116
273426
  if (buildSubstitutesAGiven(persistSource)) {
273117
273427
  if (origin2 === "preaggregate") {
273118
273428
  recordEligibilityRefused("preaggregate_over_dynamic_source");
@@ -273163,44 +273473,6 @@ function unboundParameterNames(persistSource) {
273163
273473
  }
273164
273474
  return unbound;
273165
273475
  }
273166
- function referencesPartition(persistSource) {
273167
- try {
273168
- return walkForPartition(persistSource._sourceDef, new WeakSet, 0);
273169
- } catch {
273170
- return true;
273171
- }
273172
- }
273173
- function walkForPartition(node, seen, depth) {
273174
- if (depth > MAX_GIVEN_WALK_DEPTH) {
273175
- throw new Error("partition-usage walk exceeded max depth");
273176
- }
273177
- if (node === null || typeof node !== "object")
273178
- return false;
273179
- if (seen.has(node))
273180
- return false;
273181
- seen.add(node);
273182
- if (Array.isArray(node)) {
273183
- for (const item of node) {
273184
- if (walkForPartition(item, seen, depth + 1))
273185
- return true;
273186
- }
273187
- return false;
273188
- }
273189
- const record = node;
273190
- for (const key of ["blockNotes", "notes"]) {
273191
- const arr = record[key];
273192
- if (!Array.isArray(arr))
273193
- continue;
273194
- const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
273195
- if (containsPartitionAnnotationTag(texts))
273196
- return true;
273197
- }
273198
- for (const value of Object.values(record)) {
273199
- if (walkForPartition(value, seen, depth + 1))
273200
- return true;
273201
- }
273202
- return false;
273203
- }
273204
273476
  function referencesAuthorize(persistSource) {
273205
273477
  try {
273206
273478
  return walkForAuthorize(persistSource._sourceDef, new WeakSet, 0);
@@ -273302,7 +273574,6 @@ var init_materialization_eligibility = __esm(() => {
273302
273574
  init_materialization_metrics();
273303
273575
  init_authorize();
273304
273576
  init_build_plan();
273305
- init_partition_annotation();
273306
273577
  init_persist_dynamic_terms();
273307
273578
  });
273308
273579
 
@@ -274165,9 +274436,11 @@ function extractRunTargetSourceName(query) {
274165
274436
  }
274166
274437
  function buildSourceAliasMap(query) {
274167
274438
  const aliasOf = new Map;
274168
- const declRe = /source\s*:\s*(?:`([^`]+)`|(\w+))\s+is\s+(?:`([^`]+)`|(\w+))/g;
274439
+ const text = stripMalloyCommentsAndLiterals(query);
274440
+ const ident = String.raw`(?:\x60([^\x60]+)\x60|([\p{L}\p{N}_]+))`;
274441
+ const declRe = new RegExp(String.raw`source\s*:\s*${ident}(?:\s*\([^)]*\))?\s+is\s*\(?\s*${ident}`, "gu");
274169
274442
  let match;
274170
- while ((match = declRe.exec(query)) !== null) {
274443
+ while ((match = declRe.exec(text)) !== null) {
274171
274444
  aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
274172
274445
  }
274173
274446
  return aliasOf;
@@ -274218,8 +274491,9 @@ function stripMalloyCommentsAndLiterals(text) {
274218
274491
  }
274219
274492
  return out.join("");
274220
274493
  }
274221
- function buildDerivationBaseMap(text) {
274494
+ function buildDerivationBaseMap(query) {
274222
274495
  const basesOf = new Map;
274496
+ const text = stripMalloyCommentsAndLiterals(query);
274223
274497
  const ident = String.raw`(?:\x60([^\x60]+)\x60|([\p{L}\p{N}_]+))`;
274224
274498
  const declRe = new RegExp(String.raw`(?:source|query)\s*:\s*${ident}(?:\s*\([^)]*\))?\s+is\s*\(?\s*${ident}`, "gu");
274225
274499
  let match;
@@ -274237,7 +274511,7 @@ function buildDerivationBaseMap(text) {
274237
274511
  import {
274238
274512
  expressionIsAggregate,
274239
274513
  expressionIsAnalytic,
274240
- isJoined as isJoined2
274514
+ isJoined as isJoined3
274241
274515
  } from "@malloydata/malloy";
274242
274516
  function isAccessRestricted2(f) {
274243
274517
  return f.accessModifier != null && f.accessModifier !== "public";
@@ -274249,13 +274523,13 @@ function isView(f) {
274249
274523
  return f.type === "turtle";
274250
274524
  }
274251
274525
  function isFanoutJoin(f) {
274252
- return isJoined2(f) && f.join !== "one";
274526
+ return isJoined3(f) && f.join !== "one";
274253
274527
  }
274254
274528
  function isMeasure(f) {
274255
274529
  return expressionIsAggregate(f.expressionType) && !expressionIsAnalytic(f.expressionType);
274256
274530
  }
274257
274531
  function isGroupableDimension(f) {
274258
- if (isJoined2(f) || isView(f))
274532
+ if (isJoined3(f) || isView(f))
274259
274533
  return false;
274260
274534
  if (f.expressionType === undefined)
274261
274535
  return true;
@@ -274275,7 +274549,7 @@ function validateGrainDimension(sourceName, measure, source, grainDimension) {
274275
274549
  if (!field) {
274276
274550
  return violation("unknown_grain_dimension", `Measure \`${measure}\` declares \`#@ preaggregate\` at a grain naming \`${grainDimension}\`, but \`${head}\` is not a field of \`${sourceName}\`. Use a dimension that exists on the source; a rollup can only group by the source's own dimensions.`);
274277
274551
  }
274278
- if (isJoined2(field)) {
274552
+ if (isJoined3(field)) {
274279
274553
  return violation("grain_path_through_join", `Measure \`${measure}\` declares a grain of \`${grainDimension}\`, which reaches through the join \`${head}\`. Pre-aggregating across a join is not supported in this version; copy the value into a dimension on \`${sourceName}\` and group by that instead.`);
274280
274554
  }
274281
274555
  if (parts.length > 1) {
@@ -274324,7 +274598,7 @@ function validateSourcePreaggregation(sourceName, source) {
274324
274598
  if (!declaration.declared)
274325
274599
  continue;
274326
274600
  declaredFields.push(field);
274327
- if (isJoined2(field)) {
274601
+ if (isJoined3(field)) {
274328
274602
  violations.push({
274329
274603
  code: "misplaced_on_join",
274330
274604
  sourceName,
@@ -274467,7 +274741,7 @@ var init_preaggregation_validation = __esm(() => {
274467
274741
 
274468
274742
  // src/service/source_extraction.ts
274469
274743
  import {
274470
- isJoined as isJoined3,
274744
+ isJoined as isJoined4,
274471
274745
  isSourceDef as isSourceDef6,
274472
274746
  modelDefToModelInfo
274473
274747
  } from "@malloydata/malloy";
@@ -274521,7 +274795,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274521
274795
  considerNoteOwner(authorizeNoteDeclaredBy, note, struct);
274522
274796
  }
274523
274797
  }
274524
- nearMissAuthorize.push(...collectAuthorizeNearMisses([
274798
+ nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
274525
274799
  ...ownLevelNotes(struct.annotations),
274526
274800
  ...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
274527
274801
  ].map((note) => note.text)));
@@ -274539,23 +274813,29 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274539
274813
  considerNoteOwner(authorizeNoteDeclaredBy, note, entry);
274540
274814
  }
274541
274815
  }
274542
- nearMissAuthorize.push(...collectAuthorizeNearMisses(ownLevelNotes(entry.annotations).map((note) => note.text)));
274816
+ nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes(ownLevelNotes(entry.annotations).map((note) => note.text)));
274543
274817
  }
274544
274818
  for (const struct of derivedStructsReachable(sweptStructs, modelDef)) {
274545
- nearMissAuthorize.push(...collectAuthorizeNearMisses([
274819
+ nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
274546
274820
  ...ownLevelNotes(struct.annotations),
274547
274821
  ...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
274548
274822
  ].map((note) => note.text)));
274549
274823
  }
274550
274824
  {
274551
274825
  const folded = modelAnnotations(modelDef);
274552
- nearMissAuthorize.push(...collectAuthorizeNearMisses([...folded.notes ?? [], ...folded.blockNotes ?? []].map((note) => note.text)));
274826
+ nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([...folded.notes ?? [], ...folded.blockNotes ?? []].map((note) => note.text)));
274553
274827
  }
274554
274828
  assertNoAuthorizeNearMisses(nearMissAuthorize);
274555
274829
  const authorizeOwnNotes = new Map;
274556
274830
  const attributedAuthorizeOwnNotes = new Map;
274557
- if (containsAuthorizeAnnotationTag((modelAnnotations(modelDef).notes ?? []).map((note) => note.text))) {
274558
- misplacedAuthorize.push({ kind: "file" });
274831
+ {
274832
+ const fileNoteTexts = ownLevelNoteTexts(modelAnnotations(modelDef));
274833
+ if (containsAuthorizeAnnotationTag(fileNoteTexts)) {
274834
+ misplacedAuthorize.push({
274835
+ kind: "file",
274836
+ route: fileNoteTexts.map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
274837
+ });
274838
+ }
274559
274839
  }
274560
274840
  const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef6(obj)).map((sourceObj) => {
274561
274841
  const struct = sourceObj;
@@ -274594,17 +274874,32 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274594
274874
  }
274595
274875
  }
274596
274876
  const ownNotes = ownLevelNoteTexts(struct.annotations);
274597
- const ownGates = collectAuthorizeExprs(ownNotes);
274598
274877
  const ownAuthorizeNotes = ownLevelNotes(struct.annotations).filter((note) => containsAuthorizeAnnotationTag([note.text]));
274599
- authorizeOwnNotes.set(sourceName, ownAuthorizeNotes);
274600
- attributedAuthorizeOwnNotes.set(sourceName, ownAuthorizeNotes.filter((note) => authorizeNoteDeclaredBy.get(note) === struct));
274601
- const inheritedGroups = ownGates.length === 0 ? effectiveAncestorGateExprs(struct, modelDef) : [];
274602
- const effectiveGroups = ownGates.length > 0 ? [ownGates] : inheritedGroups;
274603
- let authorize;
274604
- if (effectiveGroups.length > 0) {
274605
- authorizeMap.set(sourceName, effectiveGroups);
274606
- authorize = effectiveGroups.flat();
274607
- }
274878
+ const ownNotesByRoute = new Map;
274879
+ const attributedOwnNotesByRoute = new Map;
274880
+ for (const route of CANONICAL_AUTHORIZE_ROUTES) {
274881
+ const notesForRoute = ownAuthorizeNotes.filter((note) => authorizeAnnotationRoute(note.text) === route);
274882
+ ownNotesByRoute.set(route, notesForRoute);
274883
+ attributedOwnNotesByRoute.set(route, notesForRoute.filter((note) => authorizeNoteDeclaredBy.get(note) === struct));
274884
+ }
274885
+ authorizeOwnNotes.set(sourceName, ownNotesByRoute);
274886
+ attributedAuthorizeOwnNotes.set(sourceName, attributedOwnNotesByRoute);
274887
+ const groupsForSource = [];
274888
+ const routeGroupsByRoute = new Map;
274889
+ for (const route of CANONICAL_AUTHORIZE_ROUTES) {
274890
+ const ownExprsForRoute = collectAuthorizeExprsForRoute(ownNotes, route);
274891
+ const groupsForRoute = ownExprsForRoute.length > 0 ? [ownExprsForRoute] : effectiveAncestorGateExprs(struct, modelDef, route);
274892
+ for (const exprs of groupsForRoute) {
274893
+ groupsForSource.push({ route, exprs });
274894
+ }
274895
+ routeGroupsByRoute.set(route, groupsForRoute);
274896
+ }
274897
+ if (groupsForSource.length > 0) {
274898
+ authorizeMap.set(sourceName, groupsForSource);
274899
+ }
274900
+ const routeGroupsToFlatWire = (groups) => groups && groups.length > 0 ? groups.flat() : undefined;
274901
+ const authorize = routeGroupsToFlatWire(routeGroupsByRoute.get(AUTHORIZE_ROUTE));
274902
+ const accessFilter = routeGroupsToFlatWire(routeGroupsByRoute.get(ACCESS_FILTER_ROUTE));
274608
274903
  const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
274609
274904
  name: turtle.as || turtle.name,
274610
274905
  annotations: annotationTexts(turtle.annotations)
@@ -274617,7 +274912,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274617
274912
  continue;
274618
274913
  }
274619
274914
  const fieldName2 = field.as || field.name;
274620
- if (isJoined3(field) && isSourceDef6(field)) {
274915
+ if (isJoined4(field) && isSourceDef6(field)) {
274621
274916
  const joinedStruct = field;
274622
274917
  if (joinFieldNamesUnresolvableDeclaration(joinedStruct, modelDef)) {
274623
274918
  continue;
@@ -274625,14 +274920,16 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274625
274920
  misplacedAuthorize.push({
274626
274921
  kind: "field",
274627
274922
  name: sourceName,
274628
- fieldName: fieldName2
274923
+ fieldName: fieldName2,
274924
+ route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
274629
274925
  });
274630
274926
  continue;
274631
274927
  }
274632
274928
  misplacedAuthorize.push({
274633
274929
  kind: "field",
274634
274930
  name: sourceName,
274635
- fieldName: fieldName2
274931
+ fieldName: fieldName2,
274932
+ route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
274636
274933
  });
274637
274934
  }
274638
274935
  return {
@@ -274641,7 +274938,8 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
274641
274938
  views,
274642
274939
  filters,
274643
274940
  givens,
274644
- authorize
274941
+ authorize,
274942
+ accessFilter
274645
274943
  };
274646
274944
  });
274647
274945
  return {
@@ -274658,7 +274956,8 @@ function extractQueriesFromModelDef(modelDef) {
274658
274956
  const namedQueries = Object.values(modelDef.contents).filter(isNamedQuery);
274659
274957
  const misplacedAuthorize = namedQueries.filter((queryObj) => containsAuthorizeAnnotationTag(ownLevelNoteTexts(queryObj.annotations))).map((queryObj) => ({
274660
274958
  kind: "query",
274661
- name: queryObj.as || queryObj.name
274959
+ name: queryObj.as || queryObj.name,
274960
+ route: ownLevelNoteTexts(queryObj.annotations).map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
274662
274961
  }));
274663
274962
  const queries = namedQueries.map((queryObj) => ({
274664
274963
  name: queryObj.as || queryObj.name,
@@ -274670,10 +274969,113 @@ function extractQueriesFromModelDef(modelDef) {
274670
274969
  var init_source_extraction = __esm(() => {
274671
274970
  init_annotations();
274672
274971
  init_authorize();
274972
+ init_authorize_routes();
274673
274973
  init_filter();
274674
274974
  init_gate_registry_walk();
274675
274975
  });
274676
274976
 
274977
+ // src/service/authorize_lock.ts
274978
+ function decideLock(conditionExpr, givenNameOf, givens) {
274979
+ const ctx = { givenNameOf, givens, unresolvable: false };
274980
+ try {
274981
+ if (admits(conditionExpr, ctx, 0))
274982
+ return "admit";
274983
+ } catch {
274984
+ ctx.unresolvable = true;
274985
+ }
274986
+ return ctx.unresolvable ? "unresolvable" : "deny";
274987
+ }
274988
+ function admits(node, ctx, depth) {
274989
+ if (depth > 64 || node === null || typeof node !== "object") {
274990
+ ctx.unresolvable = true;
274991
+ return false;
274992
+ }
274993
+ const n = node;
274994
+ switch (n.node) {
274995
+ case "()":
274996
+ return admits(n.e, ctx, depth + 1);
274997
+ case "true":
274998
+ return true;
274999
+ case "false":
275000
+ return false;
275001
+ case "and": {
275002
+ const left = admits(n.kids?.left, ctx, depth + 1);
275003
+ const right = admits(n.kids?.right, ctx, depth + 1);
275004
+ return left && right;
275005
+ }
275006
+ case "=": {
275007
+ const { left, right } = n.kids ?? {};
275008
+ const pair = givenAndLiteral(left, right) ?? givenAndLiteral(right, left);
275009
+ if (!pair) {
275010
+ ctx.unresolvable = true;
275011
+ return false;
275012
+ }
275013
+ const value = resolve4(pair.given, ctx);
275014
+ if (typeof value !== "string") {
275015
+ ctx.unresolvable = true;
275016
+ return false;
275017
+ }
275018
+ return value === pair.literal;
275019
+ }
275020
+ case "inGiven": {
275021
+ if (n.not === true)
275022
+ return false;
275023
+ const given = asGiven(n.givenRef);
275024
+ const literal = asStringLiteral(n.e);
275025
+ if (!given || literal === undefined) {
275026
+ ctx.unresolvable = true;
275027
+ return false;
275028
+ }
275029
+ const value = resolve4(given, ctx);
275030
+ if (!Array.isArray(value)) {
275031
+ ctx.unresolvable = true;
275032
+ return false;
275033
+ }
275034
+ return value.some((item) => item === literal);
275035
+ }
275036
+ default:
275037
+ ctx.unresolvable = true;
275038
+ return false;
275039
+ }
275040
+ }
275041
+ function givenAndLiteral(a, b) {
275042
+ const given = asGiven(a);
275043
+ const literal = asStringLiteral(b);
275044
+ if (!given || literal === undefined)
275045
+ return;
275046
+ return { given, literal };
275047
+ }
275048
+ function asGiven(node) {
275049
+ if (node === null || typeof node !== "object")
275050
+ return;
275051
+ const n = node;
275052
+ return n.node === "given" ? n : undefined;
275053
+ }
275054
+ function asStringLiteral(node) {
275055
+ if (node === null || typeof node !== "object")
275056
+ return;
275057
+ const n = node;
275058
+ if (n.node !== "stringLiteral")
275059
+ return;
275060
+ return typeof n.literal === "string" ? n.literal : undefined;
275061
+ }
275062
+ function resolve4(given, ctx) {
275063
+ if (typeof given.id !== "string") {
275064
+ ctx.unresolvable = true;
275065
+ return;
275066
+ }
275067
+ const name = ctx.givenNameOf(given.id);
275068
+ if (name === undefined) {
275069
+ ctx.unresolvable = true;
275070
+ return;
275071
+ }
275072
+ if (!Object.prototype.hasOwnProperty.call(ctx.givens, name)) {
275073
+ ctx.unresolvable = true;
275074
+ return;
275075
+ }
275076
+ return ctx.givens[name];
275077
+ }
275078
+
274677
275079
  // src/service/model.ts
274678
275080
  import {
274679
275081
  API,
@@ -274695,6 +275097,22 @@ import { readFileSync as readFileSync2 } from "fs";
274695
275097
  import { createRequire as createRequire2 } from "module";
274696
275098
  import * as path10 from "path";
274697
275099
  import { fileURLToPath as fileURLToPath6 } from "url";
275100
+ function denyUnlessAdmitted(resolution, givens, label) {
275101
+ if (resolution.shape === "lock") {
275102
+ const outcome = decideLock(resolution.condition.e, (id) => resolution.givenNamesById.get(id), givens);
275103
+ if (outcome === "admit") {
275104
+ recordLockDecision("admitted");
275105
+ return;
275106
+ }
275107
+ recordLockDecision(outcome === "unresolvable" ? "denied_unresolvable" : "denied_by_lock");
275108
+ throw new AccessDeniedError(`Access denied for source "${label}".`);
275109
+ }
275110
+ recordLockDecision("denied_unresolvable");
275111
+ recordRowLevelGateDecision("denied_by_gate");
275112
+ if (resolution.cause)
275113
+ recordRowLevelGateRejected(resolution.cause);
275114
+ throw new AccessDeniedError(`Access denied for source "${label}".`);
275115
+ }
274698
275116
  function quoteMalloyIdentifier2(name) {
274699
275117
  return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
274700
275118
  }
@@ -274782,6 +275200,7 @@ var init_model = __esm(() => {
274782
275200
  init_annotations();
274783
275201
  init_build_plan();
274784
275202
  init_authorize();
275203
+ init_authorize_routes();
274785
275204
  init_dashboard();
274786
275205
  init_gate_dimension();
274787
275206
  init_filter();
@@ -274793,7 +275212,6 @@ var init_model = __esm(() => {
274793
275212
  init_query_metadata();
274794
275213
  init_preaggregation_validation();
274795
275214
  init_gate_registry_walk();
274796
- init_partition_annotation();
274797
275215
  init_gate_classification();
274798
275216
  init_source_extraction();
274799
275217
  init_authorize_metrics();
@@ -274867,9 +275285,17 @@ var init_model = __esm(() => {
274867
275285
  for (const source of this.sources ?? []) {
274868
275286
  if (!source.name)
274869
275287
  continue;
274870
- const exprs = this.entryPointGatesBySource.get(source.name)?.flatMap((g) => g.exprs);
274871
- if (exprs && exprs.length > 0)
274872
- source.authorize = exprs;
275288
+ const gates = this.entryPointGatesBySource.get(source.name);
275289
+ const exprsForRoute = (route) => {
275290
+ const exprs = gates?.filter((g) => g.route === route).flatMap((g) => g.exprs);
275291
+ return exprs && exprs.length > 0 ? exprs : undefined;
275292
+ };
275293
+ const lockExprs = exprsForRoute(AUTHORIZE_ROUTE);
275294
+ if (lockExprs)
275295
+ source.authorize = lockExprs;
275296
+ const filterExprs = exprsForRoute(ACCESS_FILTER_ROUTE);
275297
+ if (filterExprs)
275298
+ source.accessFilter = filterExprs;
274873
275299
  }
274874
275300
  try {
274875
275301
  this.authorizeReferencedGivenNames = this.computeAuthorizeReferencedGivenNames();
@@ -274940,6 +275366,9 @@ var init_model = __esm(() => {
274940
275366
  getAuthorize(sourceName) {
274941
275367
  return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
274942
275368
  }
275369
+ getAccessFilter(sourceName) {
275370
+ return this.sources?.find((source) => source.name === sourceName)?.accessFilter ?? [];
275371
+ }
274943
275372
  filterGivensToModelSurface(givens) {
274944
275373
  if (!givens)
274945
275374
  return givens;
@@ -275025,28 +275454,7 @@ var init_model = __esm(() => {
275025
275454
  structs.push(...derivedStructsReachable(structs, modelDef));
275026
275455
  return structs;
275027
275456
  }
275028
- anyPartitionNote;
275029
- hasAnyPartitionNote() {
275030
- if (this.anyPartitionNote !== undefined)
275031
- return this.anyPartitionNote;
275032
- this.anyPartitionNote = (() => {
275033
- const modelDef = this.modelDef;
275034
- if (!modelDef)
275035
- return false;
275036
- try {
275037
- for (const struct of this.reachableStructsForNoteSweep(modelDef)) {
275038
- if (containsPartitionAnnotationTag(annotationTexts(struct.annotations) ?? [])) {
275039
- return true;
275040
- }
275041
- }
275042
- return false;
275043
- } catch {
275044
- return true;
275045
- }
275046
- })();
275047
- return this.anyPartitionNote;
275048
- }
275049
- async assertAuthorized(sourceName, _givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
275457
+ async assertAuthorized(sourceName, givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
275050
275458
  if (bypassAuthorize) {
275051
275459
  this.noteAuthorizeBypass("source", sourceName);
275052
275460
  return;
@@ -275057,10 +275465,7 @@ var init_model = __esm(() => {
275057
275465
  const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
275058
275466
  if (resolution.shape === "row_level")
275059
275467
  continue;
275060
- recordRowLevelGateDecision("denied_by_gate");
275061
- if (resolution.cause)
275062
- recordRowLevelGateRejected(resolution.cause);
275063
- throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
275468
+ denyUnlessAdmitted(resolution, givens, entry.label);
275064
275469
  }
275065
275470
  return;
275066
275471
  }
@@ -275095,7 +275500,7 @@ var init_model = __esm(() => {
275095
275500
  const onDiskGates = this.entryPointGatesBySource.get(ownSourceName);
275096
275501
  if (onDiskGates) {
275097
275502
  const SEP = "\x1F";
275098
- const keyOf = (entry) => `${entry.label}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
275503
+ const keyOf = (entry) => `${entry.label}${SEP}${entry.route}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
275099
275504
  const byKey = new Map(entryPointGates.map((entry) => [keyOf(entry), entry]));
275100
275505
  for (const entry of onDiskGates) {
275101
275506
  byKey.set(keyOf(entry), entry);
@@ -275115,7 +275520,7 @@ var init_model = __esm(() => {
275115
275520
  return this.rowLevelFilteredRunnables.has(runnable);
275116
275521
  }
275117
275522
  async probeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate = false) {
275118
- const { entryPointGates, modelDef, struct } = await this.collectAuthorizeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate);
275523
+ const { entryPointGates, modelDef } = await this.collectAuthorizeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate);
275119
275524
  const rowLevel = [];
275120
275525
  for (const entry of entryPointGates) {
275121
275526
  const resolution = modelDef ? await this.resolveGateShape(entry, modelDef, graftScope) : { shape: "rejected", cause: undefined };
@@ -275129,20 +275534,7 @@ var init_model = __esm(() => {
275129
275534
  });
275130
275535
  continue;
275131
275536
  }
275132
- recordRowLevelGateDecision("denied_by_gate");
275133
- if (resolution.cause)
275134
- recordRowLevelGateRejected(resolution.cause);
275135
- throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
275136
- }
275137
- try {
275138
- rowLevel.push(...await resolvePartitionGraftEntries(struct, modelDef, graftScope, this.gateClassificationDeps()));
275139
- } catch (err) {
275140
- recordRowLevelGateDecision("denied_by_gate");
275141
- logger.debug("Partition filter could not be resolved; denying", {
275142
- modelPath: this.modelPath,
275143
- error: err instanceof Error ? err.message : String(err)
275144
- });
275145
- throw new AccessDeniedError(`Access denied for source "${struct?.as ?? struct?.name ?? "unknown"}".`);
275537
+ denyUnlessAdmitted(resolution, givens, entry.label);
275146
275538
  }
275147
275539
  return rowLevel;
275148
275540
  }
@@ -275156,12 +275548,12 @@ var init_model = __esm(() => {
275156
275548
  if (compositeResolvedSourceDef) {
275157
275549
  gates.push(...this.collectEntryPointGates(compositeResolvedSourceDef, modelDef, seen, true, undefined, struct ? ownLevelNotes(struct.annotations) : []));
275158
275550
  }
275159
- return gates.length > 0 || resolveEntryPointPartitions(struct, modelDef).length > 0;
275551
+ return gates.length > 0;
275160
275552
  } catch {
275161
275553
  return true;
275162
275554
  }
275163
275555
  }
275164
- async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query) {
275556
+ async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query, givens, graftScope) {
275165
275557
  if (!this.declaresAnyGate())
275166
275558
  return;
275167
275559
  const entryPoint = await this.resolveAuthorizeSourceFromRunnable(runnable);
@@ -275170,15 +275562,51 @@ var init_model = __esm(() => {
275170
275562
  if (this.entryPointGatesBySource.has(entryPoint))
275171
275563
  return;
275172
275564
  const proof = this.requestChainProvesUngated(entryPoint, buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query)));
275173
- if (proof.proven)
275565
+ if (proof.proven) {
275566
+ for (const base of proof.gatedBases) {
275567
+ for (const entry of this.entryPointGatesBySource.get(base) ?? []) {
275568
+ const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
275569
+ if (resolution.shape === "row_level") {
275570
+ this.denyLaunderedEntryPoint(entryPoint, "row_filter_not_graftable", base);
275571
+ }
275572
+ denyUnlessAdmitted(resolution, givens, entryPoint);
275573
+ }
275574
+ }
275174
275575
  return;
275576
+ }
275577
+ this.denyLaunderedEntryPoint(entryPoint, proof.reason, proof.at);
275578
+ }
275579
+ async assertLocksOnRequestDeclaredBases(entryPoint, query, givens, bypassAuthorize) {
275580
+ if (!this.declaresAnyGate())
275581
+ return;
275582
+ const basesOf = buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query));
275583
+ const seen = new Set;
275584
+ const worklist = entryPoint ? [entryPoint] : [...basesOf.keys()];
275585
+ for (let i = 0;i < worklist.length; i++) {
275586
+ const name = worklist[i];
275587
+ if (seen.has(name))
275588
+ continue;
275589
+ seen.add(name);
275590
+ if (seen.size > REQUEST_CHAIN_MAX_NAMES)
275591
+ return;
275592
+ if (this.entryPointGatesBySource.has(name)) {
275593
+ try {
275594
+ await this.assertAuthorized(name, givens, bypassAuthorize);
275595
+ } catch (error) {
275596
+ if (error instanceof AccessDeniedError) {
275597
+ this.assertQueryBoundaryEarly(name, undefined, undefined);
275598
+ }
275599
+ throw error;
275600
+ }
275601
+ continue;
275602
+ }
275603
+ for (const base of basesOf.get(name) ?? [])
275604
+ worklist.push(base);
275605
+ }
275606
+ }
275607
+ denyLaunderedEntryPoint(entryPoint, reason, at) {
275175
275608
  recordRowLevelGateDecision("denied_by_gate");
275176
- logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", {
275177
- modelPath: this.modelPath,
275178
- entryPoint,
275179
- reason: proof.reason,
275180
- at: proof.at
275181
- });
275609
+ logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", { modelPath: this.modelPath, entryPoint, reason, at });
275182
275610
  throw new AccessDeniedError(`Access denied for source "${entryPoint}".`);
275183
275611
  }
275184
275612
  declaresAnyGateMemo;
@@ -275188,6 +275616,7 @@ var init_model = __esm(() => {
275188
275616
  }
275189
275617
  requestChainProvesUngated(entryPoint, basesOf) {
275190
275618
  const seen = new Set;
275619
+ const gatedBases = [];
275191
275620
  const worklist = [entryPoint];
275192
275621
  for (let i = 0;i < worklist.length; i++) {
275193
275622
  const name = worklist[i];
@@ -275199,13 +275628,8 @@ var init_model = __esm(() => {
275199
275628
  }
275200
275629
  const modelGates = this.entryPointGatesBySource.get(name);
275201
275630
  if (modelGates !== undefined) {
275202
- if (modelGates.length > 0) {
275203
- return {
275204
- proven: false,
275205
- reason: "reaches_gated_source",
275206
- at: name
275207
- };
275208
- }
275631
+ if (modelGates.length > 0)
275632
+ gatedBases.push(name);
275209
275633
  continue;
275210
275634
  }
275211
275635
  const bases = basesOf.get(name);
@@ -275215,7 +275639,7 @@ var init_model = __esm(() => {
275215
275639
  for (const base of bases)
275216
275640
  worklist.push(base);
275217
275641
  }
275218
- return { proven: true };
275642
+ return { proven: true, gatedBases };
275219
275643
  }
275220
275644
  async authorizeAndBindRunnable(runnable, givens, options) {
275221
275645
  if (options?.bypassAuthorize) {
@@ -275226,7 +275650,7 @@ var init_model = __esm(() => {
275226
275650
  const rowLevel = await this.probeEntryPointGates(runnable, givens, graftScope, options?.skipOwnSourceGate ?? false);
275227
275651
  if (rowLevel.length === 0) {
275228
275652
  if (options?.callerQueryText && !options?.skipOwnSourceGate) {
275229
- await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText);
275653
+ await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText, givens, graftScope);
275230
275654
  }
275231
275655
  return runnable;
275232
275656
  }
@@ -275291,7 +275715,7 @@ var init_model = __esm(() => {
275291
275715
  }
275292
275716
  async resolveGateShape(entry, originModelDef, graftScope) {
275293
275717
  const result = await resolveGateShape(entry, originModelDef, graftScope, this.gateClassificationDeps());
275294
- if (result.shape === "row_level") {
275718
+ if (result.shape !== "rejected") {
275295
275719
  for (const name of result.givenNames) {
275296
275720
  this.authorizeReferencedGivenNames.add(name);
275297
275721
  }
@@ -275432,7 +275856,11 @@ var init_model = __esm(() => {
275432
275856
  return false;
275433
275857
  }
275434
275858
  async assertAuthorizedForText(text, givens) {
275435
- await this.assertAuthorized(extractRunTargetSourceName(text), givens);
275859
+ const target = extractRunTargetSourceName(text);
275860
+ await this.assertAuthorized(target, givens);
275861
+ if (!hasCallerAuthorizeAnnotation(text)) {
275862
+ await this.assertLocksOnRequestDeclaredBases(target ?? "", text, givens);
275863
+ }
275436
275864
  }
275437
275865
  async assertAuthorizedForRunnable(runnable, givens) {
275438
275866
  await this.assertAuthorizedForAllSources(runnable, givens, false, {
@@ -275490,7 +275918,7 @@ var init_model = __esm(() => {
275490
275918
  filterMap = sourceResult.filterMap;
275491
275919
  const queryResult = Model.getQueries(modelDef);
275492
275920
  queries = queryResult.queries;
275493
- assertPartitionAnnotationsValid(modelDef);
275921
+ assertNoRetiredRouteMarkers(collectRetiredRouteMarkers(modelDef));
275494
275922
  assertNoMisplacedAuthorizeAnnotations([
275495
275923
  ...sourceResult.misplacedAuthorize,
275496
275924
  ...queryResult.misplacedAuthorize
@@ -275498,20 +275926,20 @@ var init_model = __esm(() => {
275498
275926
  const legacyStringGates = findLegacyStringGates(sourceResult.authorizeOwnNotes);
275499
275927
  legacyStringGates.forEach(() => recordRowLevelGateRejected("legacy_string_gate"));
275500
275928
  assertNoLegacyStringGate(legacyStringGates);
275501
- assertAtMostOneAuthorizeGate(findMultipleAuthorizeGates(sourceResult.authorizeOwnNotes));
275929
+ assertAuthorizeGrammarValid(modelDef, sourceResult.authorizeMap, sourceResult.authorizeOwnNotes, computeGivenDeclaredTypes(givens), (_sourceName, route) => recordAuthorizeAdmitAllGate(route), sourceResult.attributedAuthorizeOwnNotes);
275502
275930
  const compiledModelDef = modelDef;
275503
275931
  await validateAuthorizeProbes(modelMaterializer, {
275504
275932
  authorizeMap: sourceResult.authorizeMap,
275505
275933
  authorizeOwnNotes: sourceResult.attributedAuthorizeOwnNotes,
275506
275934
  onRowLevelGateRejected: recordRowLevelGateRejected,
275507
- onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(authorize) gate not expressible at this entry point; every query against it will be denied", { packageName, modelPath, sourceName, detail }),
275508
- onOwnRowLevelConditionCompiled: (sourceName, condition) => {
275935
+ onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(access_filter) gate not expressible at this entry point; every query against it will be denied", { packageName, modelPath, sourceName, detail }),
275936
+ onOwnRowLevelConditionCompiled: (sourceName, condition, route) => {
275509
275937
  const struct = compiledModelDef.contents[sourceName];
275510
275938
  if (!struct || !isSourceDef7(struct))
275511
275939
  return;
275512
- validateSourceLineGateGivenUsage(sourceName, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
275940
+ validateSourceLineGateGivenUsage(sourceName, route, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
275513
275941
  recordRowLevelGateRejected(cause);
275514
- logger.warn("Row-level #(authorize) gate warning", {
275942
+ logger.warn(`#(${route}) gate warning`, {
275515
275943
  packageName,
275516
275944
  modelPath,
275517
275945
  sourceName,
@@ -275647,7 +276075,12 @@ var init_model = __esm(() => {
275647
276075
  getDashboardModelFacts() {
275648
276076
  if (!this.modelDef)
275649
276077
  return;
275650
- return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [[source.name, source.authorize ?? []]] : [])));
276078
+ return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [
276079
+ [
276080
+ source.name,
276081
+ gateGivenSource(this.sources ?? [], source.name) ?? []
276082
+ ]
276083
+ ] : [])));
275651
276084
  }
275652
276085
  hasEmptyDiscoverySurface() {
275653
276086
  if (!this.discoveryCurationEnabled)
@@ -275682,17 +276115,15 @@ var init_model = __esm(() => {
275682
276115
  return this.admittedByPackage(name, this.queryBoundary.packageCuratedSources);
275683
276116
  }
275684
276117
  isCuratedQuery(name) {
275685
- if ((this.getQueries() ?? []).some((q) => q.name === name))
276118
+ if (!this.isNotebook() && (this.getQueries() ?? []).some((q) => q.name === name))
275686
276119
  return true;
275687
276120
  return this.admittedByPackage(name, this.queryBoundary.packageCuratedQueries);
275688
276121
  }
275689
276122
  assertQueryBoundaryEarly(sourceName, queryName, query) {
275690
- if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
275691
- return "cleared";
275692
276123
  const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
275693
276124
  if (mode === "all" || !exploresDeclared)
275694
276125
  return "cleared";
275695
- if (!isQueryEntryPoint) {
276126
+ if (!isQueryEntryPoint && !this.isNotebook()) {
275696
276127
  throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
275697
276128
  }
275698
276129
  if (queryName && !query) {
@@ -275710,18 +276141,16 @@ var init_model = __esm(() => {
275710
276141
  if (query) {
275711
276142
  const target = extractRunTargetSourceName(query);
275712
276143
  if (target && !this.isCuratedSource(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
275713
- throw new NotQueryableError(`No queryable source "${target}".`);
276144
+ throw new NotQueryableError("Query target is not queryable.");
275714
276145
  }
275715
276146
  }
275716
276147
  return "deferred";
275717
276148
  }
275718
276149
  assertQueryBoundaryCompiled(compiledSource, query) {
275719
- if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
275720
- return;
275721
276150
  const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
275722
276151
  if (mode === "all" || !exploresDeclared)
275723
276152
  return;
275724
- if (!isQueryEntryPoint) {
276153
+ if (!isQueryEntryPoint && !this.isNotebook()) {
275725
276154
  throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
275726
276155
  }
275727
276156
  if (compiledSource) {
@@ -275738,7 +276167,12 @@ var init_model = __esm(() => {
275738
276167
  return;
275739
276168
  this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
275740
276169
  }
276170
+ isNotebook() {
276171
+ return this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
276172
+ }
275741
276173
  ownCuratedSourceNames() {
276174
+ if (this.isNotebook())
276175
+ return new Set;
275742
276176
  return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
275743
276177
  }
275744
276178
  derivesFromCurated(name, query) {
@@ -276127,6 +276561,9 @@ var init_model = __esm(() => {
276127
276561
  const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || (surfaceName && !this.sources?.some((s) => s.name === surfaceName) ? this.queries?.find((q) => q.name === surfaceName)?.sourceName : undefined) || surfaceName;
276128
276562
  if (earlySource) {
276129
276563
  await this.assertAuthorized(earlySource, givens ?? {}, bypassAuthorize);
276564
+ if (query && !hasCallerAuthorizeAnnotation(query) && !this.entryPointGatesBySource.has(earlySource)) {
276565
+ await this.assertLocksOnRequestDeclaredBases(earlySource, query, givens ?? {}, bypassAuthorize);
276566
+ }
276130
276567
  }
276131
276568
  try {
276132
276569
  for (const [field, callerText] of [
@@ -276175,7 +276612,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
276175
276612
  liveRunnable = runnable;
276176
276613
  const storageRoutingPossible = getPersistStorageMode() === "on" && this.serveBindings.length > 0 && !!this.serveDestinationConfig;
276177
276614
  const preaggServeMaterializer = this.preaggregateServeMaterializer;
276178
- const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize && (this.hasAnyAuthorizeNote() || this.hasAnyPartitionNote()) && await this.queryEntryPointHasRowLevelGate(runnable);
276615
+ const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize && this.hasAnyAuthorizeNote() && await this.queryEntryPointHasRowLevelGate(runnable);
276179
276616
  if (routingBlockedByRowLevelGate) {
276180
276617
  recordStorageServeRouting("blocked_by_row_level_gate");
276181
276618
  }
@@ -277659,12 +278096,12 @@ var require_recursive_readdir = __commonJS((exports, module) => {
277659
278096
  ignores = [];
277660
278097
  }
277661
278098
  if (!callback) {
277662
- return new Promise(function(resolve4, reject2) {
278099
+ return new Promise(function(resolve5, reject2) {
277663
278100
  readdir3(path11, ignores || [], function(err, data) {
277664
278101
  if (err) {
277665
278102
  reject2(err);
277666
278103
  } else {
277667
- resolve4(data);
278104
+ resolve5(data);
277668
278105
  }
277669
278106
  });
277670
278107
  });
@@ -284918,7 +285355,7 @@ var require_uri_all = __commonJS((exports, module) => {
284918
285355
  target.fragment = relative5.fragment;
284919
285356
  return target;
284920
285357
  }
284921
- function resolve5(baseURI, relativeURI, options) {
285358
+ function resolve6(baseURI, relativeURI, options) {
284922
285359
  var schemelessOptions = assign({ scheme: "null" }, options);
284923
285360
  return serialize(resolveComponents(parse(baseURI, schemelessOptions), parse(relativeURI, schemelessOptions), schemelessOptions, true), schemelessOptions);
284924
285361
  }
@@ -285186,7 +285623,7 @@ var require_uri_all = __commonJS((exports, module) => {
285186
285623
  exports2.removeDotSegments = removeDotSegments;
285187
285624
  exports2.serialize = serialize;
285188
285625
  exports2.resolveComponents = resolveComponents;
285189
- exports2.resolve = resolve5;
285626
+ exports2.resolve = resolve6;
285190
285627
  exports2.normalize = normalize2;
285191
285628
  exports2.equal = equal;
285192
285629
  exports2.escapeComponent = escapeComponent;
@@ -285552,20 +285989,20 @@ var require_resolve2 = __commonJS((exports, module) => {
285552
285989
  var util5 = require_util12();
285553
285990
  var SchemaObject = require_schema_obj();
285554
285991
  var traverse = require_json_schema_traverse();
285555
- module.exports = resolve5;
285556
- resolve5.normalizeId = normalizeId;
285557
- resolve5.fullPath = getFullPath;
285558
- resolve5.url = resolveUrl;
285559
- resolve5.ids = resolveIds;
285560
- resolve5.inlineRef = inlineRef;
285561
- resolve5.schema = resolveSchema;
285562
- function resolve5(compile, root, ref) {
285992
+ module.exports = resolve6;
285993
+ resolve6.normalizeId = normalizeId;
285994
+ resolve6.fullPath = getFullPath;
285995
+ resolve6.url = resolveUrl;
285996
+ resolve6.ids = resolveIds;
285997
+ resolve6.inlineRef = inlineRef;
285998
+ resolve6.schema = resolveSchema;
285999
+ function resolve6(compile, root, ref) {
285563
286000
  var refVal = this._refs[ref];
285564
286001
  if (typeof refVal == "string") {
285565
286002
  if (this._refs[refVal])
285566
286003
  refVal = this._refs[refVal];
285567
286004
  else
285568
- return resolve5.call(this, compile, root, refVal);
286005
+ return resolve6.call(this, compile, root, refVal);
285569
286006
  }
285570
286007
  refVal = refVal || this._schemas[ref];
285571
286008
  if (refVal instanceof SchemaObject) {
@@ -285788,7 +286225,7 @@ var require_resolve2 = __commonJS((exports, module) => {
285788
286225
 
285789
286226
  // ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/error_classes.js
285790
286227
  var require_error_classes = __commonJS((exports, module) => {
285791
- var resolve5 = require_resolve2();
286228
+ var resolve6 = require_resolve2();
285792
286229
  module.exports = {
285793
286230
  Validation: errorSubclass(ValidationError),
285794
286231
  MissingRef: errorSubclass(MissingRefError)
@@ -285803,8 +286240,8 @@ var require_error_classes = __commonJS((exports, module) => {
285803
286240
  };
285804
286241
  function MissingRefError(baseId, ref, message) {
285805
286242
  this.message = message || MissingRefError.message(baseId, ref);
285806
- this.missingRef = resolve5.url(baseId, ref);
285807
- this.missingSchema = resolve5.normalizeId(resolve5.fullPath(this.missingRef));
286243
+ this.missingRef = resolve6.url(baseId, ref);
286244
+ this.missingSchema = resolve6.normalizeId(resolve6.fullPath(this.missingRef));
285808
286245
  }
285809
286246
  function errorSubclass(Subclass) {
285810
286247
  Subclass.prototype = Object.create(Error.prototype);
@@ -286346,7 +286783,7 @@ var require_validate4 = __commonJS((exports, module) => {
286346
286783
 
286347
286784
  // ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/index.js
286348
286785
  var require_compile2 = __commonJS((exports, module) => {
286349
- var resolve5 = require_resolve2();
286786
+ var resolve6 = require_resolve2();
286350
286787
  var util5 = require_util12();
286351
286788
  var errorClasses = require_error_classes();
286352
286789
  var stableStringify = require_fast_json_stable_stringify();
@@ -286406,7 +286843,7 @@ var require_compile2 = __commonJS((exports, module) => {
286406
286843
  RULES,
286407
286844
  validate: validateGenerator,
286408
286845
  util: util5,
286409
- resolve: resolve5,
286846
+ resolve: resolve6,
286410
286847
  resolveRef,
286411
286848
  usePattern,
286412
286849
  useDefault,
@@ -286445,7 +286882,7 @@ var require_compile2 = __commonJS((exports, module) => {
286445
286882
  return validate;
286446
286883
  }
286447
286884
  function resolveRef(baseId2, ref, isRoot) {
286448
- ref = resolve5.url(baseId2, ref);
286885
+ ref = resolve6.url(baseId2, ref);
286449
286886
  var refIndex = refs[ref];
286450
286887
  var _refVal, refCode;
286451
286888
  if (refIndex !== undefined) {
@@ -286462,11 +286899,11 @@ var require_compile2 = __commonJS((exports, module) => {
286462
286899
  }
286463
286900
  }
286464
286901
  refCode = addLocalRef(ref);
286465
- var v2 = resolve5.call(self2, localCompile, root, ref);
286902
+ var v2 = resolve6.call(self2, localCompile, root, ref);
286466
286903
  if (v2 === undefined) {
286467
286904
  var localSchema = localRefs && localRefs[ref];
286468
286905
  if (localSchema) {
286469
- v2 = resolve5.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
286906
+ v2 = resolve6.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
286470
286907
  }
286471
286908
  }
286472
286909
  if (v2 === undefined) {
@@ -289983,7 +290420,7 @@ var require_data2 = __commonJS((exports, module) => {
289983
290420
  // ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/ajv.js
289984
290421
  var require_ajv = __commonJS((exports, module) => {
289985
290422
  var compileSchema = require_compile2();
289986
- var resolve5 = require_resolve2();
290423
+ var resolve6 = require_resolve2();
289987
290424
  var Cache = require_cache();
289988
290425
  var SchemaObject = require_schema_obj();
289989
290426
  var stableStringify = require_fast_json_stable_stringify();
@@ -290075,7 +290512,7 @@ var require_ajv = __commonJS((exports, module) => {
290075
290512
  var id = this._getId(schema);
290076
290513
  if (id !== undefined && typeof id != "string")
290077
290514
  throw new Error("schema id must be string");
290078
- key = resolve5.normalizeId(key || id);
290515
+ key = resolve6.normalizeId(key || id);
290079
290516
  checkUnique(this, key);
290080
290517
  this._schemas[key] = this._addSchema(schema, _skipValidation, _meta, true);
290081
290518
  return this;
@@ -290121,7 +290558,7 @@ var require_ajv = __commonJS((exports, module) => {
290121
290558
  }
290122
290559
  }
290123
290560
  function _getSchemaFragment(self2, ref) {
290124
- var res = resolve5.schema.call(self2, { schema: {} }, ref);
290561
+ var res = resolve6.schema.call(self2, { schema: {} }, ref);
290125
290562
  if (res) {
290126
290563
  var { schema, root, baseId } = res;
290127
290564
  var v = compileSchema.call(self2, schema, root, undefined, baseId);
@@ -290137,7 +290574,7 @@ var require_ajv = __commonJS((exports, module) => {
290137
290574
  }
290138
290575
  }
290139
290576
  function _getSchemaObj(self2, keyRef) {
290140
- keyRef = resolve5.normalizeId(keyRef);
290577
+ keyRef = resolve6.normalizeId(keyRef);
290141
290578
  return self2._schemas[keyRef] || self2._refs[keyRef] || self2._fragments[keyRef];
290142
290579
  }
290143
290580
  function removeSchema(schemaKeyRef) {
@@ -290165,7 +290602,7 @@ var require_ajv = __commonJS((exports, module) => {
290165
290602
  this._cache.del(cacheKey);
290166
290603
  var id = this._getId(schemaKeyRef);
290167
290604
  if (id) {
290168
- id = resolve5.normalizeId(id);
290605
+ id = resolve6.normalizeId(id);
290169
290606
  delete this._schemas[id];
290170
290607
  delete this._refs[id];
290171
290608
  }
@@ -290190,14 +290627,14 @@ var require_ajv = __commonJS((exports, module) => {
290190
290627
  if (cached2)
290191
290628
  return cached2;
290192
290629
  shouldAddSchema = shouldAddSchema || this._opts.addUsedSchema !== false;
290193
- var id = resolve5.normalizeId(this._getId(schema));
290630
+ var id = resolve6.normalizeId(this._getId(schema));
290194
290631
  if (id && shouldAddSchema)
290195
290632
  checkUnique(this, id);
290196
290633
  var willValidate = this._opts.validateSchema !== false && !skipValidation;
290197
290634
  var recursiveMeta;
290198
- if (willValidate && !(recursiveMeta = id && id == resolve5.normalizeId(schema.$schema)))
290635
+ if (willValidate && !(recursiveMeta = id && id == resolve6.normalizeId(schema.$schema)))
290199
290636
  this.validateSchema(schema, true);
290200
- var localRefs = resolve5.ids.call(this, schema);
290637
+ var localRefs = resolve6.ids.call(this, schema);
290201
290638
  var schemaObj = new SchemaObject({
290202
290639
  id,
290203
290640
  schema,
@@ -299430,7 +299867,7 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
299430
299867
  refined = true;
299431
299868
  const [, sourceName] = accessDeniedMatch;
299432
299869
  suggestions = [
299433
- `Suggestion: Access to source '${sourceName}' is restricted by an #(authorize) gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
299870
+ `Suggestion: Access to source '${sourceName}' is restricted by an authorize gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
299434
299871
  ];
299435
299872
  } else if (viewNotFoundMatch) {
299436
299873
  refined = true;
@@ -300544,12 +300981,15 @@ var REASON_BY_UNAVAILABLE = {
300544
300981
  "too-many-entities": "too-many-entities",
300545
300982
  error: "provider-error"
300546
300983
  };
300547
- function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false) {
300984
+ function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false, droppedSources = new Set) {
300548
300985
  const bySource = new Map;
300549
300986
  const cardFor = (name, modelPathFallback) => {
300550
300987
  if (!name)
300551
300988
  return;
300552
300989
  const key = sourceContextKey(modelPathFallback, name);
300990
+ if (droppedSources.has(key)) {
300991
+ return;
300992
+ }
300553
300993
  let entry = bySource.get(key);
300554
300994
  if (!entry) {
300555
300995
  const ctx = sourceContext.get(key);
@@ -300564,6 +301004,7 @@ function toSourceResults(results, sourceContext, environmentName, packageName, s
300564
301004
  ...ctx?.oneLineSummary ? { one_line_summary: ctx.oneLineSummary } : {},
300565
301005
  ...ctx?.doc ? { docs: ctx.doc } : {},
300566
301006
  ...ctx?.givens ? { givens: ctx.givens } : {},
301007
+ ...ctx?.accessFilter ? { accessFilter: ctx.accessFilter } : {},
300567
301008
  ...ctx?.authorize ? { authorize: ctx.authorize } : {},
300568
301009
  ...ctx?.filters ? { filter_params: ctx.filters } : {},
300569
301010
  joins: ctx?.joins ?? []
@@ -300684,7 +301125,7 @@ function projectEntity(e, environmentName, packageName) {
300684
301125
  }
300685
301126
  function finishRanked(args) {
300686
301127
  const windowed = windowBySource(args.rows, args.max);
300687
- const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode);
301128
+ const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode, args.droppedSources);
300688
301129
  return {
300689
301130
  sources,
300690
301131
  totalSources: windowed.totalSources,
@@ -300931,10 +301372,15 @@ function collectJoinedFields(args) {
300931
301372
  });
300932
301373
  }
300933
301374
  }
301375
+ function isUnconditionalDenyAuthorize(apiSource) {
301376
+ const isDeny = (expr) => expr.trim().toLowerCase() === "false";
301377
+ return (apiSource.accessFilter ?? []).some(isDeny) || (apiSource.authorize ?? []).some(isDeny);
301378
+ }
300934
301379
  async function collectEntities(pkg) {
300935
301380
  const models = [...await pkg.listModels()].sort((a, b) => comparePaths(a.path ?? "", b.path ?? ""));
300936
301381
  const entities = [];
300937
301382
  const governance = new Map;
301383
+ const droppedSources = new Set;
300938
301384
  const sourceTextFor = makeSourceTextReader(pkg);
300939
301385
  let n = 0;
300940
301386
  for (const apiModel of models) {
@@ -300948,10 +301394,18 @@ async function collectEntities(pkg) {
300948
301394
  const queries = model.getQueries() ?? [];
300949
301395
  const apiSources = model.getSources?.() ?? [];
300950
301396
  const modelDef = model.getModelDef?.();
301397
+ for (const apiSource of apiSources) {
301398
+ if (apiSource.name && isUnconditionalDenyAuthorize(apiSource)) {
301399
+ droppedSources.add(sourceContextKey(modelPath, apiSource.name));
301400
+ }
301401
+ }
300951
301402
  for (const sourceInfo of sourceInfos) {
300952
301403
  const sourceName = sourceInfo.name;
300953
- const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
300954
301404
  const governanceKey = sourceContextKey(modelPath, sourceName);
301405
+ if (droppedSources.has(governanceKey)) {
301406
+ continue;
301407
+ }
301408
+ const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
300955
301409
  if (!governance.has(governanceKey)) {
300956
301410
  const apiSource = apiSources.find((c) => c.name === sourceName);
300957
301411
  if (apiSource) {
@@ -300964,6 +301418,10 @@ async function collectEntities(pkg) {
300964
301418
  ...given.default !== undefined ? { default: given.default } : {}
300965
301419
  }
300966
301420
  ] : []),
301421
+ accessFilter: (apiSource.accessFilter ?? []).map((expression) => ({
301422
+ expression,
301423
+ given_names: referencedGivenNames(expression)
301424
+ })),
300967
301425
  authorize: (apiSource.authorize ?? []).map((expression) => ({
300968
301426
  expression,
300969
301427
  given_names: referencedGivenNames(expression)
@@ -301038,6 +301496,9 @@ async function collectEntities(pkg) {
301038
301496
  continue;
301039
301497
  if (!query.sourceName)
301040
301498
  continue;
301499
+ if (droppedSources.has(sourceContextKey(modelPath, query.sourceName))) {
301500
+ continue;
301501
+ }
301041
301502
  entities.push({
301042
301503
  id: String(n++),
301043
301504
  kind: "query",
@@ -301057,7 +301518,7 @@ async function collectEntities(pkg) {
301057
301518
  seen.add(key);
301058
301519
  return true;
301059
301520
  });
301060
- return { entities: collapseAliases(deduped), governance };
301521
+ return { entities: collapseAliases(deduped), governance, droppedSources };
301061
301522
  }
301062
301523
  function malloyType(field) {
301063
301524
  const kind = field.type?.kind;
@@ -301160,6 +301621,7 @@ function buildSourceContext(collected) {
301160
301621
  joins: [],
301161
301622
  ...summary ? { oneLineSummary: summary } : {},
301162
301623
  ...gates?.givens.length ? { givens: gates.givens } : {},
301624
+ ...gates?.accessFilter.length ? { accessFilter: gates.accessFilter } : {},
301163
301625
  ...gates?.authorize.length ? { authorize: gates.authorize } : {},
301164
301626
  ...gates?.filters.length ? { filters: gates.filters } : {}
301165
301627
  });
@@ -301207,7 +301669,8 @@ async function getPackageIndex(environmentStore, environmentName, packageName) {
301207
301669
  byId,
301208
301670
  index,
301209
301671
  entityCount: entities.length,
301210
- sourceContext: buildSourceContext(collected)
301672
+ sourceContext: buildSourceContext(collected),
301673
+ droppedSources: collected.droppedSources
301211
301674
  };
301212
301675
  indexCache.set(pkg, built);
301213
301676
  logger.debug("[MCP Tool getContext] Built and cached entity index", {
@@ -301225,13 +301688,13 @@ var GET_CONTEXT_DESCRIPTION = `Retrieve the entities in a Malloy package most re
301225
301688
  - Read warnings and any error/stale field before trusting a number.
301226
301689
  - A source's joins list is complete: empty means it declares none, so write that relationship inline.
301227
301690
  - Read a source's doc before querying: it carries grain and population rules its fields do not.
301228
- - authorize means gated: supply the givens it names or the query is denied.
301691
+ - accessFilter/authorize mean gated; a deny-all source never appears here.
301229
301692
 
301230
301693
  ## Parameters
301231
301694
  search_targets: one per concept, {target_type, search_text}; target_type is source|dimension|measure|view|join|dimensional_value, omitting search_text enumerates that type. scopes: {environment, package} + optional model_path, source, entity_name. limit caps sources (max 150, counted as cards). offset pages a listing. user_prompt: the question asked. include_code or a pinned entity_name returns code.
301232
301695
 
301233
301696
  ## Response
301234
- sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, authorize (report-only), filter_params. entities[] nest under it: name, entity_type, description, data_type, relationship (fan-out), join_path, aliases, relevance, entity_id. A joined field's name IS its dotted path; use it verbatim.
301697
+ sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, accessFilter/authorize, filter_params. entities[] nest under it: name, entity_type, description, data_type, relationship (fan-out), join_path, aliases, relevance, entity_id. A joined field's name IS its dotted path; use it verbatim.
301235
301698
  ranking, returned of total_available sources, next_offset on a listing, warnings[].
301236
301699
  Semantic fills relevance: no sources = nothing cleared the floor; below_cutoff_count of total_entities rejected. "lexical" adds retrieval_reason; only "indexing" is worth a retry.
301237
301700
 
@@ -301266,7 +301729,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
301266
301729
  });
301267
301730
  return contextError(buildMalloyUri({ environment: environmentName, package: packageName }, "get-context"), `${environmentName}/${packageName}`, error);
301268
301731
  }
301269
- const { byId, index, sourceContext } = pkgIndex;
301732
+ const { byId, index, sourceContext, droppedSources } = pkgIndex;
301270
301733
  const uri = buildMalloyUri({ environment: environmentName, package: packageName }, "get-context");
301271
301734
  let staleNote;
301272
301735
  try {
@@ -301304,7 +301767,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
301304
301767
  });
301305
301768
  const spendable = (rows) => sourceName ? rows.filter((e) => e.kind !== "source").length : rows.length;
301306
301769
  const cappable = spendable(inScope);
301307
- const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode);
301770
+ const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode, droppedSources);
301308
301771
  const consumed = request.offset + sources2.length;
301309
301772
  const cardsInScope = sourceName ? new Set(inScope.map((e) => sourceContextKey(e.modelPath, e.source ?? ""))).size : inScope.length;
301310
301773
  const listingEnvelope = {
@@ -301420,7 +301883,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
301420
301883
  environmentName,
301421
301884
  packageName,
301422
301885
  searchTexts: searchTextsByIndex,
301423
- includeCode: request.includeCode
301886
+ includeCode: request.includeCode,
301887
+ droppedSources
301424
301888
  });
301425
301889
  return jsonResource(uri, {
301426
301890
  sources: sources2,
@@ -301481,7 +301945,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
301481
301945
  environmentName,
301482
301946
  packageName,
301483
301947
  searchTexts: searchTextsByIndex,
301484
- includeCode: request.includeCode
301948
+ includeCode: request.includeCode,
301949
+ droppedSources
301485
301950
  });
301486
301951
  const envelope = {
301487
301952
  sources,
@@ -301747,7 +302212,7 @@ class QueryController {
301747
302212
  const p = await environment.getPackage(packageName, false);
301748
302213
  const model = p.getModel(modelPath);
301749
302214
  if (!model) {
301750
- throw new ModelNotFoundError(`${modelPath} does not exist`);
302215
+ throw new ModelNotFoundError(`No queryable model "${modelPath}".`);
301751
302216
  } else {
301752
302217
  const {
301753
302218
  result,
@@ -309865,6 +310330,23 @@ class Package {
309865
310330
  const explores = this.packageMetadata.explores;
309866
310331
  return !!(explores && explores.length > 0);
309867
310332
  }
310333
+ surfaceWidenedWarning;
310334
+ noteSurfaceChangeFrom(previousSurface) {
310335
+ this.surfaceWidenedWarning = undefined;
310336
+ if (!previousSurface || previousSurface.length === 0 || this.packageMetadata.explores !== undefined) {
310337
+ return;
310338
+ }
310339
+ const message = `This package published "${previousSurface.join('", "')}" before the last ` + `reload and publishes no surface now, so every model in it is listed ` + `and queryable by name again, including the sources that surface was ` + `withholding. A surface file that was deleted or renamed is the usual ` + `cause. If that was intended, nothing to do. If not, restore the file ` + `(under its original name -- the name IS the surface), or declare an ` + `"explores" in publisher.json naming what this package should ` + `publish. To keep the package open deliberately, write ` + `"explores": [], which says so and stops this notice.`;
310340
+ this.surfaceWidenedWarning = message;
310341
+ logger.warn(`Package ${this.packageName} no longer publishes a surface`, {
310342
+ packageName: this.packageName,
310343
+ detail: message
310344
+ });
310345
+ }
310346
+ surfaceIsIndexModel() {
310347
+ const explores = this.packageMetadata.explores;
310348
+ return this.models.has(INDEX_MODEL_NAME) && explores?.length === 1 && explores[0] === INDEX_MODEL_NAME;
310349
+ }
309868
310350
  exploreSet() {
309869
310351
  const explores = this.packageMetadata.explores;
309870
310352
  return explores && explores.length > 0 ? new Set(explores) : null;
@@ -310204,6 +310686,8 @@ class Package {
310204
310686
  ...this.droppedPersistWarnings(),
310205
310687
  ...this.persistRefusalWarnings(),
310206
310688
  ...this.emptyDiscoveryWarnings(),
310689
+ ...this.brokenSurfaceWarnings(),
310690
+ ...this.surfaceWidenedWarning !== undefined ? [{ message: this.surfaceWidenedWarning }] : [],
310207
310691
  ...this.persistenceCollisionWarnings().map((message) => ({ message })),
310208
310692
  ...this.preaggregateAccessWarnings(),
310209
310693
  ...incrementalPolicyAdvisories(this.incrementalPolicySources),
@@ -310364,7 +310848,7 @@ class Package {
310364
310848
  if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
310365
310849
  problems.push({
310366
310850
  entry,
310367
- reason: `notebooks are always public and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
310851
+ reason: `notebooks are always listed and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
310368
310852
  });
310369
310853
  } else if (!malloyModels.has(entry)) {
310370
310854
  problems.push({
@@ -310535,13 +311019,39 @@ class Package {
310535
311019
  if (model.hasEmptyDiscoverySurface()) {
310536
311020
  warnings.push({
310537
311021
  model: modelPath,
310538
- message: `Model "${modelPath}" is listed in explores but exposes ` + `nothing: its export closure surfaces no sources or named ` + `queries (typically an import-only file, or an export {} ` + `that filters everything out). Add e.g. ` + `'export { source_name }' to surface sources on this ` + `model, or remove it from explores.`
311022
+ message: `Model "${modelPath}" is on this package's discovery ` + `surface but exposes nothing: its export closure surfaces ` + `no sources or named queries (typically an import-only ` + `file, or an export {} that filters everything out). Add ` + `e.g. 'export { source_name }' to surface sources on this ` + `model, or ` + (this.surfaceIsIndexModel() ? `delete the file AND any "explores" entry naming it. ` + `Deleting the file alone returns the package to ` + `listing every model only when no "explores" was ` + `written; where one was, it would be left naming a ` + `model that no longer exists and the package would ` + `list nothing.` : `remove it from explores.`)
310539
311023
  });
310540
311024
  }
310541
311025
  }
310542
311026
  return warnings;
310543
311027
  }
311028
+ brokenSurfaceWarnings() {
311029
+ const exploreSet = this.exploreSet();
311030
+ if (!exploreSet || exploreSet.size === 0)
311031
+ return [];
311032
+ const surface = Array.from(this.models.entries()).filter(([modelPath]) => modelPath.endsWith(MODEL_FILE_SUFFIX) && exploreSet.has(modelPath));
311033
+ if (surface.length === 0)
311034
+ return [];
311035
+ if (!surface.every(([, model]) => !!model.getCompilationError())) {
311036
+ return [];
311037
+ }
311038
+ const collateral = Array.from(this.models.entries()).filter(([modelPath, model]) => modelPath.endsWith(MODEL_FILE_SUFFIX) && !exploreSet.has(modelPath) && !model.getCompilationError()).length;
311039
+ const [firstPath] = surface[0];
311040
+ const subject = surface.length === 1 ? `Model "${firstPath}" is this package's whole discovery surface ` + `and failed to compile` : `Every model on this package's discovery surface ` + `(${surface.map(([p]) => `"${p}"`).join(", ")}) failed to ` + `compile`;
311041
+ return [
311042
+ {
311043
+ model: firstPath,
311044
+ message: `${subject}, so the package exposes nothing: every model in ` + `it, including the ${collateral} that compiled, is now refused ` + `by name with a 404 that reads as "does not exist". Fix the ` + `compile error${surface.length === 1 ? "" : "s"} to restore ` + `them -- ` + surface.map(([modelPath, model]) => `${modelPath}: ` + `${model.getCompilationError()?.message ?? "unknown error"}`).join("; ")
311045
+ }
311046
+ ];
311047
+ }
310544
311048
  logEmptyDiscoveryWarnings() {
311049
+ for (const warning of this.brokenSurfaceWarnings()) {
311050
+ logger.warn(`Package ${this.packageName} has a broken discovery surface`, {
311051
+ packageName: this.packageName,
311052
+ detail: warning.message
311053
+ });
311054
+ }
310545
311055
  for (const warning of this.emptyDiscoveryWarnings()) {
310546
311056
  logger.warn(`Package ${this.packageName} has a blank-looking model`, {
310547
311057
  packageName: this.packageName,
@@ -310659,7 +311169,9 @@ class Package {
310659
311169
  await this.pushPreaggregateServeModels();
310660
311170
  this.renderTagWarnings = renderTagWarnings;
310661
311171
  this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
311172
+ const previousSurface = this.packageMetadata.explores;
310662
311173
  this.packageMetadata.explores = outcome.packageMetadata.explores;
311174
+ this.noteSurfaceChangeFrom(previousSurface);
310663
311175
  this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
310664
311176
  this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
310665
311177
  this.applyDiscoveryPolicyToModels();
@@ -310821,6 +311333,29 @@ class Package {
310821
311333
  });
310822
311334
  }
310823
311335
  }
311336
+ lintTilesAgainstSurface(modelPath, manifest) {
311337
+ const model = this.models.get(modelPath);
311338
+ if (!model || !manifest.tiles)
311339
+ return [];
311340
+ const listedModel = (this.packageMetadata.explores ?? []).find((entry) => entry.endsWith(MODEL_FILE_SUFFIX) && !isDashboardModelPath(entry));
311341
+ const importInto = listedModel ? `a listed file such as "${listedModel}"` : `a file you list there`;
311342
+ const findings = [];
311343
+ for (const tile of manifest.tiles) {
311344
+ try {
311345
+ model.assertQueryBoundaryEarly(undefined, undefined, `run: ${tile.query}`);
311346
+ } catch (error) {
311347
+ if (!(error instanceof NotQueryableError))
311348
+ throw error;
311349
+ findings.push({
311350
+ model: modelPath,
311351
+ subject: manifest.name,
311352
+ message: `tile "${tile.query}" reads a source this package's surface ` + `does not publish, so the tile answers 404 once served, ` + `although the file compiles. Listing a file publishes what ` + `it declares, not what it imports. Fix: add the file that ` + `declares the source to 'explores', keeping the entries ` + `already there, or import it into ${importInto} and add ` + `it to that file's export { … }.`,
311353
+ severity: "error"
311354
+ });
311355
+ }
311356
+ }
311357
+ return findings;
311358
+ }
310824
311359
  async lintDashboards(factsByPath, allFacts, heldBack, unconventionalSlugs, knownSlugs, droppedByError) {
310825
311360
  const warnings = [];
310826
311361
  const withheldDrills = new Map;
@@ -310854,10 +311389,11 @@ class Package {
310854
311389
  });
310855
311390
  }
310856
311391
  }
311392
+ const remedy = this.surfaceIsIndexModel() ? `This package's surface is "${INDEX_MODEL_NAME}", and a ` + `dashboard file cannot be exported from it -- dashboards are ` + `files, not sources. To serve this one, declare (or extend) ` + `an 'explores' in publisher.json listing both ` + `"${INDEX_MODEL_NAME}" and "${modelPath}"; an explicit key ` + `overrides the convention. Or set ` + `queryableSources: "all" to keep the curated surface for ` + `discovery only.` : `Add it to 'explores', or set queryableSources: "all" to ` + `keep the curated surface for discovery only.`;
310857
311393
  warnings.push({
310858
311394
  model: modelPath,
310859
311395
  subject: name,
310860
- message: `is a dashboard, but "${modelPath}" is not listed in ` + `'explores' and this package sets ` + `queryableSources: "declared", so its query would be ` + `refused. It is not served. Add it to 'explores', or set ` + `queryableSources: "all" to keep the curated surface for ` + `discovery only. Listing it is not always sufficient on its ` + `own: the queryable sources are the union of every listed ` + `file's export closure, so a tile reading a source that only ` + `an UNLISTED file exports is still refused. List that file ` + `too, or re-export the source from one already listed.`,
311396
+ message: `is a dashboard, but "${modelPath}" is not part of this ` + `package's discovery surface and this package sets ` + `queryableSources: "declared", so its query would be ` + `refused. It is not served. ${remedy} Listing it is not ` + `always sufficient on its own: the queryable sources are ` + `the union of every listed file's export closure, so a tile ` + `reading a source that only an UNLISTED file exports is ` + `still refused. List that file too, or re-export the source ` + `from one already listed.`,
310861
311397
  severity: "warn"
310862
311398
  });
310863
311399
  }
@@ -310867,6 +311403,11 @@ class Package {
310867
311403
  for (const finding of findings) {
310868
311404
  warnings.push({ model: modelPath, ...finding });
310869
311405
  }
311406
+ if (manifest) {
311407
+ for (const finding of this.lintTilesAgainstSurface(modelPath, manifest)) {
311408
+ warnings.push(finding);
311409
+ }
311410
+ }
310870
311411
  }
310871
311412
  const drillFacts = Array.from(allFacts.values());
310872
311413
  for (const finding of withheldDrills.values()) {
@@ -311741,6 +312282,7 @@ ${source}` : source ?? "";
311741
312282
  await this.rebindServeBindingsFromLocalStore(_package);
311742
312283
  if (existingPackage !== undefined && reload) {
311743
312284
  this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
312285
+ _package.noteSurfaceChangeFrom(existingPackage.getPackageMetadata().explores);
311744
312286
  }
311745
312287
  this.packages.set(packageName, _package);
311746
312288
  this.setPackageStatus(packageName, "serving" /* SERVING */);
@@ -312044,11 +312586,12 @@ ${source}` : source ?? "";
312044
312586
  const materializationBase = metadata.materialization !== undefined ? { ...metadata.materialization, ...preservedQueryMetadata } : onDiskMaterialization !== undefined ? { ...onDiskMaterialization } : undefined;
312045
312587
  const materializationBlock = resolvedScope !== undefined ? { ...materializationBase ?? {}, scope: resolvedScope } : materializationBase;
312046
312588
  const resolvedQueryMetadata = metadata.queryMetadata ?? materializationBlock?.queryMetadata ?? existingManifest.queryMetadata;
312589
+ const echoesDerivedSurface = existingManifest.explores === undefined && Array.isArray(metadata.explores) && metadata.explores.length === 1 && metadata.explores[0] === INDEX_MODEL_NAME;
312047
312590
  const updatedManifest = {
312048
312591
  ...existingManifest,
312049
312592
  name: metadata.name,
312050
312593
  description: metadata.description,
312051
- ...metadata.explores !== undefined ? { explores: metadata.explores } : {},
312594
+ ...metadata.explores !== undefined && !echoesDerivedSurface ? { explores: metadata.explores } : {},
312052
312595
  ...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
312053
312596
  ...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {},
312054
312597
  ...resolvedScope !== undefined ? { scope: resolvedScope } : {},
@@ -313578,9 +314121,9 @@ class EnvironmentStore {
313578
314121
  }
313579
314122
  const file = fs9.createWriteStream(zipFilePath);
313580
314123
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
313581
- await new Promise((resolve5, reject2) => {
314124
+ await new Promise((resolve6, reject2) => {
313582
314125
  file.on("error", reject2);
313583
- file.on("finish", resolve5);
314126
+ file.on("finish", resolve6);
313584
314127
  });
313585
314128
  await this.unzipEnvironment(zipFilePath);
313586
314129
  logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
@@ -313618,9 +314161,9 @@ class EnvironmentStore {
313618
314161
  }
313619
314162
  const file = fs9.createWriteStream(absoluteFilePath);
313620
314163
  item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
313621
- await new Promise((resolve5, reject2) => {
314164
+ await new Promise((resolve6, reject2) => {
313622
314165
  file.on("error", reject2);
313623
- file.on("finish", resolve5);
314166
+ file.on("finish", resolve6);
313624
314167
  });
313625
314168
  }));
313626
314169
  logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
@@ -313640,7 +314183,7 @@ class EnvironmentStore {
313640
314183
  await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
313641
314184
  const repoUrl = `https://github.com/${owner}/${repoName}`;
313642
314185
  const reporter = new CloneProgressReporter(cloneProgressLabel(`${owner}/${repoName}`, progressContext));
313643
- await new Promise((resolve5, reject2) => {
314186
+ await new Promise((resolve6, reject2) => {
313644
314187
  esm_default2({
313645
314188
  progress: (event) => reporter.onProgress(event)
313646
314189
  }).clone(repoUrl, absoluteDirPath, GIT_CLONE_OPTIONS, (err) => {
@@ -313655,7 +314198,7 @@ class EnvironmentStore {
313655
314198
  reject2(err);
313656
314199
  return;
313657
314200
  }
313658
- resolve5();
314201
+ resolve6();
313659
314202
  });
313660
314203
  });
313661
314204
  if (!cleanPackagePath) {
@@ -314445,7 +314988,7 @@ class Protocol {
314445
314988
  }
314446
314989
  request(request, resultSchema, options) {
314447
314990
  const { relatedRequestId, resumptionToken, onresumptiontoken } = options !== null && options !== undefined ? options : {};
314448
- return new Promise((resolve5, reject2) => {
314991
+ return new Promise((resolve6, reject2) => {
314449
314992
  var _a2, _b, _c, _d, _e, _f;
314450
314993
  if (!this._transport) {
314451
314994
  reject2(new Error("Not connected"));
@@ -314496,7 +315039,7 @@ class Protocol {
314496
315039
  }
314497
315040
  try {
314498
315041
  const result = resultSchema.parse(response.result);
314499
- resolve5(result);
315042
+ resolve6(result);
314500
315043
  } catch (error) {
314501
315044
  reject2(error);
314502
315045
  }
@@ -316647,17 +317190,17 @@ var compileShape = {
316647
317190
  source: exports_external.string().optional().describe("The Malloy source to validate; never executed. Required at scope append/file; optional at scope package (a what-if replacement for modelPath)."),
316648
317191
  scope: exports_external.enum(["append", "file", "package"]).optional().describe(`What source means. "append" (default): append to modelPath. "file": compile AS modelPath to validate an edit. "package": run reload's worker compiler over all .malloy/.malloynb files without serving the result; an optional source replaces modelPath.`),
316649
317192
  includeSql: exports_external.boolean().optional().describe("When true and the source ends in a runnable query, also return the generated SQL for inspection. The query is still not run."),
316650
- givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any #(authorize) gate on the target model, whether or not includeSql is set.")
317193
+ givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any authorize gate on the target model, whether or not includeSql is set.")
316651
317194
  };
316652
317195
  var COMPILE_DESCRIPTION = `Compile-check Malloy without running a query. Use this while authoring instead of a throwaway execute_query.
316653
317196
 
316654
317197
  ## Scopes (the scope parameter)
316655
317198
  - "append" (default): append source to modelPath. Use for NEW definitions; existing definitions report "Cannot redefine". Positions refer to the concatenated file.
316656
317199
  - "file": compile source AS modelPath. Use to validate an EDIT before saving; positions match the submitted file.
316657
- - "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and #(authorize) still gates caller text. A missing exact path is warned and treated as a new file. Save and call reload_package to serve a clean edit.
317200
+ - "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and authorize gates still apply to caller text. A missing exact path is warned and treated as a new file. Save and call reload_package to serve a clean edit.
316658
317201
 
316659
317202
  ## Parameters
316660
- - environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and #(authorize) values. Caller source may not declare #(authorize).
317203
+ - environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and authorize-gate values. Caller source may not declare an access-control gate — neither #(authorize) nor #(access_filter).
316661
317204
 
316662
317205
  ## Response
316663
317206
  { status: "success"|"error", diagnostics: [{ severity, message, code, model, line, character, endLine, endCharacter, replacement }], sql? }. Positions are 0-based; model is the package-relative file the diagnostic points at (which can be pre-existing content, not your source). status is "error" only when an error-severity diagnostic exists; errors are also stated in a plain text block.`;
@@ -318746,6 +319289,21 @@ If the agent could not reasonably have known to ask, that is a model gap.
318746
319289
  | \`GUIDANCE-NOT-RETRIEVED\` | entities came back, governing guidance did not | model: put guidance on the entities agents search for |
318747
319290
  | \`GUIDANCE-DECLINED\` | guidance was retrieved and judged inapplicable | model: state the business default, not a caveat |
318748
319291
 
319292
+ **Before diagnosing a \`NOT-RETURNED\`, check the call actually returned
319293
+ nothing.** The \`tool_call\` event carries \`retrieval_mode\` and \`rankedSummary\`.
319294
+ A \`rankedSummary\` of \`null\` means the response could not be read, not that it
319295
+ was empty -- a body too large for the model's context is written to a file, and
319296
+ the run records no summary for it. There is nothing to diagnose there: say the
319297
+ call was not measured.
319298
+
319299
+ And **never attribute a miss to the embedding index unless \`retrieval_mode\`
319300
+ says \`lexical\`.** The mode is recorded per call precisely so this is checkable.
319301
+ A diagnosis once explained a phantom empty result with "the index was not ready
319302
+ yet" on a run whose index was ready before the first question and whose
319303
+ response did hold results; the empty list was a parsing bug in the harness. An
319304
+ unverifiable cause that sounds right is worse than \`needs_human\`, because it
319305
+ closes the finding.
319306
+
318749
319307
  A missing join is coverage, not an agent-call miss. The model has to volunteer
318750
319308
  relationships. A declared join is not a retrieval entity; do not look for it in
318751
319309
  \`get_context\` results.
@@ -319791,6 +320349,27 @@ Output, exactly this shape:
319791
320349
  reading that produced it, that is \`no_match\`. Use \`near_match\` only when you
319792
320350
  can name the rubric clause that makes the difference defensible.
319793
320351
 
320352
+ **A rubric clause cannot make a wrong VALUE defensible, and a clause that
320353
+ tries is a defect in the rubric rather than a licence to you.** \`near_match\`
320354
+ turns on the answer being right under a reading the QUESTION allows -- a tie
320355
+ broken the other way, a grain the question left open, a basis the question
320356
+ never fixed. It does not turn on the answer being transparent about how it
320357
+ got a figure the question did not ask for. Those two look alike in a rubric
320358
+ and are opposites in a report: one is a number a reader can act on, the
320359
+ other is a number a reader would act on wrongly. Naming the method makes a
320360
+ wrong figure DIAGNOSABLE, which is worth having, and it is not partial
320361
+ credit.
320362
+
320363
+ The test, before you write \`near_match\` on a case with a value: would a
320364
+ reader who acted on this figure be wrong? If yes, it is \`no_match\` however
320365
+ plainly the answer explained itself, and however the rubric is worded. Say
320366
+ in \`why\` that you are overriding a rubric clause, so the clause gets fixed.
320367
+ This rule exists because a set shipped one: a question asked for sales over
320368
+ the company's season, the answer gave the meteorological window 25% lower
320369
+ and said which window it used, and a clause granting \`near_match\` for a
320370
+ stated window kept a materially wrong answer out of the pass rate
320371
+ entirely -- the arm reported 100%.
320372
+
319794
320373
  It is a third outcome because as a pass it was a large share of the measured
319795
320374
  noise: the same unchanged answer reads \`match\` in one run and \`near_match\`
319796
320375
  in the next, and the pass rate moves although nothing did. A verdict whose
@@ -320454,11 +321033,19 @@ the run measure something other than what it names:
320454
321033
  first time somebody tidies up; the set, the run directory and the write-up
320455
321034
  belong in git beside the model.
320456
321035
 
320457
- 2. The server must be up with retrieval tracing on, so a call's ranked results
320458
- can be recovered afterwards (open-source Publisher: \`PUBLISHER_MCP_TRACE=retrieval\`).
320459
- Confirm a trace lookup is available (absent means tracing is off).
320460
- Refuse to start a scored run without it: failures without traces cannot be
320461
- attributed.
321036
+ 2. The server must be up. Where your host offers retrieval tracing, turn it on
321037
+ and confirm a trace lookup answers, so a call's ranked results can be
321038
+ recovered afterwards.
321039
+
321040
+ **Open-source Publisher has none.** \`PUBLISHER_MCP_TRACE=retrieval\` is set by
321041
+ \`serve.py\` and read by nothing: the trace store was written and never merged,
321042
+ so there is no trace tool and \`traceId\` is null on every local attempt. This
321043
+ does not block a scored run, because attribution never depended on it:
321044
+ \`rankedSummary\` is copied onto the \`tool_call\` event at capture, precisely so
321045
+ the evidence survives without a store. So do not refuse a local run for want
321046
+ of tracing -- an earlier version of this rule did, and it refused every local
321047
+ run there has ever been. Refuse one whose \`tool_call\` events carry no
321048
+ \`rankedSummary\`, which is what attribution actually reads.
320462
321049
 
320463
321050
  3. Health-check: your host's status check until it reports serving, and inspect
320464
321051
  \`loadErrors\`. A dead database that still answers HTTP is an environment
@@ -320626,6 +321213,13 @@ conductor. Do not:
320626
321213
 
320627
321214
  - The model is the only thing improve edits. No question text, qids, or
320628
321215
  expected values in any name, doc, or comment.
321216
+ - **You are measuring a model, not reviewing this harness.** Read a script when
321217
+ a number you have to report cannot be explained otherwise, and stop there. Do
321218
+ not audit the scripts, propose fixes to them, or hand back tooling critique
321219
+ in place of a result -- a run that ends in harness feedback has not answered
321220
+ the question it was asked. A defect that changed a number gets one sentence
321221
+ in the report; a defect that changed nothing gets one line in chat, once.
321222
+ Fixing it is a different task, and the user starts it.
320629
321223
  - When the environment misbehaves, stop. Never diagnose a sick system. The
320630
321224
  harness is part of the environment: a known-broken measurement does not
320631
321225
  become quotable by being finished. Measured against this directive, an arm
@@ -321586,6 +322180,30 @@ one exception is anything that qualifies THIS run's number -- a truncated
321586
322180
  attempt, a contaminated one, an unestablished answer key -- which the "Eval
321587
322181
  failures" section exists for.
321588
322182
 
322183
+ **Do not audit the harness. You were asked to measure a model.** This is the
322184
+ most common way this job goes wrong, and it does not look like going wrong: a
322185
+ run turns up something odd in a script, the odd thing is genuinely a bug, and
322186
+ the reply comes back as a critique of the tooling with the model's score
322187
+ somewhere underneath. The reader asked what their model scored. Answer that.
322188
+
322189
+ So, unless the user asked you to work on the harness:
322190
+
322191
+ - Do not read harness source to satisfy your own curiosity about a number.
322192
+ Read it when a number you must report cannot be explained any other way, and
322193
+ stop when it can.
322194
+ - Do not propose harness fixes, refactors, flags or "while I was in there"
322195
+ improvements. Not in the report, not in the chat reply.
322196
+ - When a harness defect DID change this run's number, the report gets one
322197
+ sentence: what the number should be and why. Not the mechanism, not the
322198
+ file, not the fix.
322199
+ - Keep a defect that changed nothing out of the report entirely. Mention it
322200
+ once in chat, in a line, and let the user decide whether they want it
322201
+ chased.
322202
+
322203
+ A harness bug you found and did not chase is not a loose end. It is the job
322204
+ being done. If the user wants it fixed they will say so, and then it is a
322205
+ different task with its own turn.
322206
+
321589
322207
  ## Step 1: build the artifacts, before writing a word
321590
322208
 
321591
322209
  \`\`\`bash
@@ -321624,6 +322242,29 @@ not in \`~/Downloads\`, both of which have lost a findings document before.
321624
322242
  from the console.** A figure retyped from scrollback is a figure nobody can
321625
322243
  check, and the console rounds.
321626
322244
 
322245
+ ### Keep it under a page and a half
322246
+
322247
+ **Budget: about 80 lines, and 120 is the ceiling for a run with several
322248
+ distinct failures.** Reports have shipped at 200 and the length is not
322249
+ thoroughness -- it is the artifact links restated as prose, the cascade
322250
+ explained twice, and a paragraph apologising for a figure nobody disputed.
322251
+ A reader who wants the detail opens the case matrix, which is why step 1 builds
322252
+ it. What cannot be recovered from the artifacts is your judgement: what broke,
322253
+ why, and what to do. Spend the lines there.
322254
+
322255
+ What earns its place: the headline, one row per case, one short entry per
322256
+ failure, the retrieval numbers, and the next steps. What does not: restating a
322257
+ number you already gave, explaining what a cascade is before showing it,
322258
+ defending a decision nobody questioned, or any section whose content is that
322259
+ nothing happened -- except "Eval failures", where "None." is the point.
322260
+
322261
+ **Being brief is not being terse with the vocabulary.** The reader does not
322262
+ know what \`near_match\`, a cluster, recall or a holdout is, so the first time
322263
+ one appears, say what it means in the same sentence -- "\`near_match\`, which is
322264
+ excluded from the pass rate" -- and then use it. Cutting the explanations is
322265
+ how a short report becomes an unreadable one; cutting the restatements is how
322266
+ it becomes a good one.
322267
+
321627
322268
  ### The template
321628
322269
 
321629
322270
  \`\`\`markdown
@@ -321637,7 +322278,7 @@ check, and the console rounds.
321637
322278
  - Target \`<env>/<package>\`, model \`<modelPath>\`, pinned \`<modelSha or targetVersion>\`
321638
322279
  - Answerer \`<model>\`, judge \`<model>\`, cap \`<maxTurns>\` turns
321639
322280
  - Steps run: scrape/run, eval<, diagnose><, improve>
321640
- - Cost $X answerer + $Y judge
322281
+ - Cost $X answerer + $Y judge, N turns median / N p90, N entities delivered per answer
321641
322282
 
321642
322283
  ## The result
321643
322284
 
@@ -321648,9 +322289,15 @@ check, and the console rounds.
321648
322289
 
321649
322290
  [case matrix](<url>) - [aggregate tables](<url>)
321650
322291
 
321651
- ## Retrieval and coverage
322292
+ ## Coverage, retrieval, accuracy, cost
321652
322293
 
321653
- Covered? -> Retrieved? -> Correct?, with the per-arm numbers under each.
322294
+ **Coverage N of M** -- how many questions the model can express an answer to at
322295
+ all. **Entity recall N%** -- one number, first, before the cascade. It is the share
322296
+ of the entities an answer needed that retrieval actually handed the agent, and
322297
+ it is the headline of this section for the same reason the pass rate is the
322298
+ headline of the last one. Then the cascade: Covered? -> Retrieved? -> Correct?,
322299
+ with the per-arm numbers under each. Say in one clause what recall counts, then
322300
+ give the number.
321654
322301
 
321655
322302
  ## Model failures
321656
322303
 
@@ -321679,10 +322326,10 @@ these in order and put every one that fires into the list, with its command:
321679
322326
 
321680
322327
  | If the run shows | Then the next step is |
321681
322328
  |---|---|
321682
- | \`coverage: unmeasured\` | run \`check_coverage.py --set <set> --model <pkg> --out coverage.json\`, then re-run with \`--coverage\` so it charges the failures |
322329
+ | \`coverage: unmeasured\` | the run was given \`--no-coverage\`, or the measurement failed and said so. It is measured by default, so say in "Eval failures" that this score cannot tell a model gap from a bad answer, and re-run without the flag |
321683
322330
  | \`goldenCheck: skipped\` or the set names no \`truthPackage\` | build one with \`init_truth_package.py\`; until then the goldens were derived through the model under test and certify themselves |
321684
322331
  | any golden still \`provisional\` | re-derive and \`verify_goldens.py --promote\` |
321685
- | a stale entity name warning | fix \`expectedEntities\`; it scores as a retrieval miss on every run until you do |
322332
+ | a stale entity name warning | fix \`expectedEntities\`; it scores as a retrieval miss on every run until you do. A next step, not a section: it goes in this list and nowhere else in the report |
321686
322333
  | a passing case with recall below 1.0 | check whether \`required\` over-specifies one path |
321687
322334
  | \`truncated\` non-empty | re-run those cases at a higher cap with \`--from\` |
321688
322335
  | diagnose did not run | run it, or say the failures have no owner yet |
@@ -321725,8 +322372,23 @@ wrong answer. One entry each: what the answer said, what was right, and the
321725
322372
  mechanism in one sentence. Do not write a cluster id here; write what it got
321726
322373
  wrong.
321727
322374
 
321728
- **Eval failures** are everything that stopped the run measuring the model.
321729
- Report every one that occurred, with its count and its qids, and say plainly
322375
+ **Eval failures** are the things that stopped the run measuring the model, and
322376
+ **only** those. The test is one question: *did this cost a verdict, or make one
322377
+ untrustworthy?* If no, it does not appear in the report at any length.
322378
+
322379
+ That rules out most of what is tempting to put here, and all of it has been put
322380
+ here on a real run: a \`-dirty\` model pin, a stale-entity-name warning that cost
322381
+ no case, a defect you hit in the harness and worked around, a setup step that
322382
+ took two tries, anything you would open with "worth knowing, though it changes
322383
+ no verdict". A reader wants to know what their model scored. Harness defects are
322384
+ real and belong in a harness issue, filed against the harness -- that is the
322385
+ skill's opening rule, and this section is where it gets broken.
322386
+
322387
+ The section is usually two words. "**None.**" is a complete and good answer, and
322388
+ a reader who sees it learns exactly what they need to. Do not pad it into a
322389
+ paragraph explaining the absence.
322390
+
322391
+ Report every one that DID occur, with its count and its qids, and say plainly
321730
322392
  that these are NOT evidence about the model:
321731
322393
 
321732
322394
  | What happened | How it reads in the ledger | Who fixes it |
@@ -321745,11 +322407,32 @@ harness prints \`INCOMPLETE\` and withholds the percentage; the report does the
321745
322407
  same. Give the counts and the re-run command instead of a number with a caveat,
321746
322408
  because the number is what gets repeated and the caveat is what gets dropped.
321747
322409
 
321748
- ## Retrieval and coverage belong in every report
322410
+ ## The four measurements, and why a report carries all of them
322411
+
322412
+ A pass rate says an answer was wrong. It does not say where, and on its own it
322413
+ makes every failure look like the model's. A run measures four things, in this
322414
+ order, and each one tells you whether the next is even a fair question:
322415
+
322416
+ | | Measures | A failure here means |
322417
+ |---|---|---|
322418
+ | **a. Coverage** | can the data and the model express an answer at all | nothing downstream was winnable. Fix the model |
322419
+ | **b. Retrieval** | did \`get_context\` hand the agent the entities it needed | the model has it and the agent never saw it: the docs, or the search wording |
322420
+ | **c. Accuracy** | did the agent get the answer right | it had what it needed and still missed: the agent, or a doc that misleads |
322421
+ | **d. Cost** | dollars, turns, wall-clock, entities per answer | it works and cannot be afforded, which is its own kind of not working |
321749
322422
 
321750
- A pass rate says an answer was wrong. It does not say WHERE, and the three
321751
- metrics that do are already in the run summary and the notebook. Report them,
321752
- because a report that omits them makes every failure look like the model's:
322423
+ **Coverage first, and a report without it is incomplete.** A model that cannot
322424
+ express an answer can never succeed at that question, so a pass rate quoted
322425
+ without coverage cannot distinguish a bad model from an unanswerable set.
322426
+ \`run_baseline.py\` measures it by default; if a run skipped it, say so in
322427
+ "Eval failures" and treat the score as provisional.
322428
+
322429
+ **Cost is a result, not an aside.** Report dollars, the turn and call counts,
322430
+ and the entity payload per answer. An agent that answers correctly in 40 turns
322431
+ and 75 delivered entities per question is a different product from one that
322432
+ does it in 8 and 5, and only the report says which you have.
322433
+
322434
+ Report them all, because a report that omits them makes every failure look
322435
+ like the model's:
321753
322436
 
321754
322437
  **Copy the cascade the run printed. Do not re-derive it.** \`cascade_lines()\`
321755
322438
  in \`run_baseline.py\` already renders every rung, and it carries one field a
@@ -321965,6 +322648,7 @@ This table is a catalogue of what exists, not of what is loaded. A host that ins
321965
322648
  | \`skill:malloy-notebooks\` | Building Malloy notebooks (.malloynb) |
321966
322649
  | \`skill:malloy-analysis-report\` | Combining validated queries into a notebook report or dashboard |
321967
322650
  | \`skill:malloy-analysis-pitfalls\` | Checking a query and its results before presenting an answer |
322651
+ | \`skill:malloy-source-unreachable\` | A source is missing from discovery, or a query was refused with a 404 or 403 |
321968
322652
  | \`malloy-notebook-chat\` | The chat is bound to a notebook or saved report; answer from its cells. Ships in \`analysis\`. |
321969
322653
  | \`skill:malloy-phrase-detection\` | Turning a plain-English question into search targets for the context tool |
321970
322654
 
@@ -322021,7 +322705,7 @@ Call \`search_malloy_docs\` with your question. Use \`skill:malloy-patterns\` to
322021
322705
  },
322022
322706
  {
322023
322707
  name: "malloy-analysis",
322024
- description: "The workflow for answering a data question over a Malloy semantic model - a metric, breakdown, trend or chart - from get_context discovery through execute_query to verified delivery.",
322708
+ description: "Read before your first get_context call on any business or data question. The required workflow, not background: which source, which rows, how to verify. Skipping it causes confident wrong answers.",
322025
322709
  body: `# Malloy analysis workflow
322026
322710
 
322027
322711
  > **Tool names** are written bare here - \`get_context\`, \`execute_query\`, \`search_malloy_docs\`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
@@ -322131,7 +322815,7 @@ End with a short **Next steps**: one or two specific deeper analyses the data co
322131
322815
  {
322132
322816
  name: "malloy-analysis-pitfalls",
322133
322817
  description: "Checks to run on a query and its numbers before a result is presented - wrong grain, fan-out, invented field names, mismatched filters, count vs distinct. The common data-analysis pitfalls.",
322134
- body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats = null }\n}\n```\n\nAn agent once refused a correct answer it had already computed because `count(aircraft.tail_num)` returned 36 against 17,875 flights; measured at the base grain, zero flights were missing a seat value.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
322818
+ body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats is null }\n}\n```\n\nAn agent once refused a correct answer it had already computed because `count(aircraft.tail_num)` returned 36 against 17,875 flights; measured at the base grain, zero flights were missing a seat value.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
322135
322819
  },
322136
322820
  {
322137
322821
  name: "malloy-analysis-report",
@@ -322430,6 +323114,13 @@ Scanned at a glance is a dashboard; read top to bottom is a notebook.
322430
323114
 
322431
323115
  ## Build sequence
322432
323116
 
323117
+ 0. **CHECK THE PACKAGE CAN SERVE A DASHBOARD AT ALL.** If its root holds an \`index.malloy\`, that
323118
+ file is the package's surface, and a dashboard's file is not something it can \`export\`, so the
323119
+ dashboard you are about to write will compile and then be withheld. Every package made by the
323120
+ scaffolder has that file, so this is the common case, not the corner one. Serving dashboards
323121
+ there means declaring an \`explores\` in \`publisher.json\` naming \`index.malloy\` and every dashboard
323122
+ file, which overrides the convention. Settle that first: it is a different curation shape for the
323123
+ package, not a line to add at the end. See "Read the lint" for the warning it produces otherwise.
322433
323124
  1. **READ THE MODEL FIRST.** Get the real source, view, dimension, and given names from the package:
322434
323125
  \`get_context\` if you have it, otherwise the REST model endpoint or the \`.malloy\` files.
322435
323126
  Never guess a name. A guessed field in a query fails the whole package load, not just that one
@@ -322840,8 +323531,8 @@ Package warnings after a reload are the dashboard's test suite. Fix all of them:
322840
323531
  Findings carry a \`severity\`, but \`warn\` is the ordinary default and tells you nothing about how bad
322841
323532
  one is. Read the text, not the severity and not the count. One message is worth recognising because
322842
323533
  it changes what the rest of the list means: **"Dashboard lint stopped early, so this list is
322843
- incomplete"**. A dashboard withheld from \`explores\` also loses its own findings, so a short list for a
322844
- withheld file is not a clean bill of health.
323534
+ incomplete"**. A dashboard withheld from the package's surface also loses its own findings, so a
323535
+ short list for a withheld file is not a clean bill of health.
322845
323536
 
322846
323537
  **Read the status the reload itself returns, not the listing.** One dashboard that fails to compile
322847
323538
  fails the whole package load, and the reload answers **424** with the compile error. A package that
@@ -322856,11 +323547,22 @@ at all appears there too, without \`stale\`, and is absent from the listing enti
322856
323547
 
322857
323548
  If the reload is 200 and the others are listed but yours is not, discovery skipped the file instead,
322858
323549
  usually a missing or misspelled \`# artifact\` tag, which is the same mechanism that deliberately skips
322859
- an untagged shared include. There is a second cause if the package's \`publisher.json\` carries an
322860
- \`explores\` list: a dashboard whose file is missing from it is withheld rather than served, and the
322861
- warning says so and names the fix. The list is what matters, not the \`queryableSources\` setting, which
322862
- is \`declared\` by default; a package with no \`explores\` list withholds nothing. Where there is one, a
322863
- \`suggest\` source has to be queryable as well as resolvable, so it needs to be on the list too.
323550
+ an untagged shared include. There is a second cause whenever the package has a curated surface: a
323551
+ dashboard whose file is off it is withheld rather than served, and the warning says so and names the
323552
+ fix. A package has a surface in two ways, and the second is easy to miss because no manifest field
323553
+ records it:
323554
+
323555
+ - the package's \`publisher.json\` carries an \`explores\` list, and the dashboard's file is not on it;
323556
+ - the package root holds an **\`index.malloy\`**, which IS the surface. This bites in both
323557
+ directions, and a dashboard's file is not something an \`index.malloy\` exports either way: adding
323558
+ that file to a package that has dashboards withholds every one of them, and adding a dashboard to
323559
+ a package that already has one (every scaffolded package does) means it is never served. The fix
323560
+ is the same in both directions, and it is not an edit to \`index.malloy\`: declare an \`explores\` in
323561
+ \`publisher.json\` naming \`index.malloy\` and every dashboard file.
323562
+
323563
+ Either way the surface is what matters, not the \`queryableSources\` setting, which is \`declared\` by
323564
+ default; a package with neither withholds nothing. Where there is a surface, a \`suggest\` source has
323565
+ to be queryable as well as resolvable, so it needs to be reachable through it too.
322864
323566
 
322865
323567
  **A clean reload is not proof the tags are right.** The checks above read names and resolve them; the
322866
323568
  separate warning for a tag that does not _parse_ is syntax only: it carries no
@@ -323229,7 +323931,7 @@ A confirmed source architecture and a confirmed set of field definitions (rename
323229
323931
  {
323230
323932
  name: "malloy-gotchas-queries",
323231
323933
  description: "Common Malloy query and view mistakes, for writing views, queries or notebooks - chart constraints, aggregate filters, joined field aliasing, method syntax, time truncation vs extraction.",
323232
- body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Query Clauses Are Newline-Separated\n\nDo not use trailing commas between query clauses. Each clause goes on its own line.\n\n```malloy\n// WRONG: trailing comma before limit\nrun: source -> { group_by: status, aggregate: n is count(), limit: 10 }\n// RIGHT: newline-separated\nrun: source -> {\n group_by: status\n aggregate: n is count()\n limit: 10\n}\n```\n\nClauses: `group_by:`, `aggregate:`, `nest:`, `order_by:`, `limit:`, `where:`, `having:`, `select:`, `calculate:`\n\n## Fields Within a Clause: Commas or Newlines, Never Semicolons\n\nSemicolons are not a separator anywhere in Malloy. Multiple fields under one `aggregate:` / `group_by:` are separated by commas (inline) or newlines (one per line); a `;` fails with `no viable alternative at input '<next-field>'` pointing at the field right after it.\n\n```malloy\n// WRONG: semicolons between fields\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT: commas inline...\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n// ...or newlines\nrun: schools -> {\n aggregate:\n total is count()\n charters is count() { where: is_charter }\n}\n```"
323934
+ body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## A Measure Is Already an Aggregate\n\n```malloy\n// WRONG: \"Aggregate expression cannot be aggregate\" - flight_count is a measure\nrun: flights -> { aggregate: busiest is max(flight_count) }\n// RIGHT: aggregate per group, then take the max in a second stage\nrun: flights -> { group_by: carrier, aggregate: n is flight_count } -> { aggregate: busiest is max(n) }\n```\n\nThe message does not name the field. When it appears, look for a model measure you wrapped in another aggregate. If you only need the top row, skip the second stage: `order_by: n desc` with `limit: 1`.\n\n## Window Braces Bind to the Function\n\n```malloy\n// WRONG: \"`partition_by` is not supported for this kind of expression\"\ncalculate: share is sum_cumulative(n) / region_total { partition_by: region, order_by: n desc }\n// RIGHT: the brace goes directly after the window function call\ncalculate: share is sum_cumulative(n) { partition_by: region, order_by: n desc } / region_total\n```\n\nFor a within-group share, compute the denominator in `aggregate:` (`region_total is all(count(), region)`). A window's `{ partition_by }` does not apply to `all()`.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Strings: Apostrophes and Concatenation\n\n```malloy\n// WRONG: \"no viable alternative at input 's'\" - the apostrophe ends the literal\nwhere: name = 'Mac's Diner'\n// RIGHT\nwhere: name = \"Mac's Diner\"\n\n// WRONG: \"unexpected '+'\" / \"no viable alternative at input '||'\"\ngroup_by: route is origin ++ '-' ++ destination\n// RIGHT: there is no concatenation operator\ngroup_by: route is concat(origin, '-', destination)\n```\n\n## A Semicolon Inside a Clause Ends It\n\nBetween clauses a newline, comma or `;` all work. Within one clause, separate fields with commas or newlines: a `;` closes the clause, and the next field is orphaned.\n\n```malloy\n// WRONG: \"no viable alternative at input 'charters'\"\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n```\n\nThe same message appears for an unnamed aggregate after the first: `aggregate: n, max(x)` fails at `max`. Name every entry.\n\n## Compiles, Then Refused by the Warehouse\n\n`Query execution failed: ...` means the Malloy compiled and the warehouse rejected the SQL, often over a type. The position it quotes is in the generated SQL, not in your query. BigQuery, for example, will not partition a window function on a FLOAT64 column. `partition_by:` takes only a field name, so cast in `group_by:` (`season_key is season::string`) and partition on `season_key`, or fix the type in the model."
323233
323935
  },
323234
323936
  {
323235
323937
  name: "malloy-gotchas-rendering",
@@ -323362,7 +324064,7 @@ view: overview is {
323362
324064
  {
323363
324065
  name: "malloy-html-data-app-runtime",
323364
324066
  description: "The JavaScript that drives an in-package HTML data app - calling Publisher.query, building queries from filter state, handling results and errors. For writing the page's data code.",
323365
- body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a source with `#(authorize)`, a bad given (unknown name or wrong-typed value) fails closed in the authorize check, so it looks like access denied rather than validation. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
324067
+ body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a gated source, a bad given (unknown name or wrong-typed value) fails closed, so it looks like access denied rather than validation. A `#(authorize)` lock also answers 403 for a caller it simply does not admit. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
323366
324068
  },
323367
324069
  {
323368
324070
  name: "malloy-html-data-apps",
@@ -324006,7 +324708,7 @@ LookML flattens nested field names with double underscores: \`project__id\`, \`l
324006
324708
  {
324007
324709
  name: "malloy-lookml-review/curate-visibility",
324008
324710
  description: "LookML Visibility → Malloy Access Modifiers (Step 8). Reference detail for the malloy-lookml-review skill.",
324009
- body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control). Masking and `#(authorize)` are independent layers; use the mask for \"everyone sees a coarsened value,\" `#(authorize)` for \"only some callers see the field at all.\"\n\n## Pattern: Long→wide custom-field pivot\n\nSee `build-derived-tables.md` → \"Long→wide entity-values pivot\" for turning an entity-attribute-value (EAV) custom-field table into wide columns with **filtered aggregates**, the right move when LookML modeled N per-attribute joins.\n\n## Process\n\n1. Read the Visibility Seeds notes captured during discovery\n2. For each field, apply the classification rules above\n3. Present the classified list to the user for confirmation before applying access modifiers"
324711
+ body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control). `#(authorize)` refuses a caller it does not admit with a 403; `#(access_filter)` is the row-scoping sibling. Masking and these gates are independent layers; use the mask for \"everyone sees a coarsened value,\" a gate for \"only some callers see the field at all.\"\n\n## Pattern: Long→wide custom-field pivot\n\nSee `build-derived-tables.md` → \"Long→wide entity-values pivot\" for turning an entity-attribute-value (EAV) custom-field table into wide columns with **filtered aggregates**, the right move when LookML modeled N per-attribute joins.\n\n## Process\n\n1. Read the Visibility Seeds notes captured during discovery\n2. For each field, apply the classification rules above\n3. Present the classified list to the user for confirmation before applying access modifiers"
324010
324712
  },
324011
324713
  {
324012
324714
  name: "malloy-lookml-review/discover",
@@ -324234,12 +324936,12 @@ Present coverage in this order:
324234
324936
  {
324235
324937
  name: "malloy-materialization",
324236
324938
  description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.",
324237
- body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting an `#(authorize)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **An `#(authorize)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting an `#(authorize)`-gated source**."
324939
+ body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting a `#(access_filter)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **A `#(access_filter)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting a `#(access_filter)`-gated source**."
324238
324940
  },
324239
324941
  {
324240
324942
  name: "malloy-materialization-tuning",
324241
324943
  description: "Optimize a package's Malloy Persistence materializations for cost and performance using the malloy-pub CLI and the materialization history. Recommend what to persist, what to stop persisting, and how to schedule/scope it. Use when the user asks to make a package cheaper or faster, tune persistence, decide what to materialize, or review persist/schedule choices.",
324242
- body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries an `#(authorize)` gate, check which case it is first. A gate reached only through a `join_*`, or inherited from a base it cannot be attributed to, makes the colocated persist **refuse with a 422** - so do not recommend persisting it at all. Where the gate is provably the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `skill:malloy-materialization`, which also covers where that window does and does not bind).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n ```bash\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n ```bash\n malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n ```bash\n malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg> # 5-field UTC cron\n malloy-pub schedule clear --environment <env> --package <pkg>\n ```\n\n `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
324944
+ body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries a `#(access_filter)` gate, check which case it is first. A gate reached only through a `join_*`, or inherited from a base it cannot be attributed to, makes the colocated persist **refuse with a 422** - so do not recommend persisting it at all. Where the gate is provably the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `skill:malloy-materialization`, which also covers where that window does and does not bind).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n ```bash\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n ```bash\n malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n ```bash\n malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg> # 5-field UTC cron\n malloy-pub schedule clear --environment <env> --package <pkg>\n ```\n\n `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
324243
324945
  },
324244
324946
  {
324245
324947
  name: "malloy-model",
@@ -324461,7 +325163,7 @@ A given is **declared bare** but **referenced with a \`$\` sigil** in expression
324461
325163
  - **Givens don't auto-inject a \`where:\`.** Unlike \`#(filter)\`, you write the filter expression that references the given yourself (e.g. \`where: dimension ~ $given_name\`).
324462
325164
  - **Not every filter maps cleanly.** A filter with no neutral match-all literal default - e.g. a scalar date/number range like \`> @2020-01-01\` - is not a good \`given:\`; keep those on \`#(filter)\`. Two more cases keep using \`#(filter)\`: mandatory scoping filters (\`required\`) and system-injected row-level filters (\`implicit\`), both below.
324463
325165
 
324464
- Givens are also the substrate for access control - see "Access Control: Source Gating with \`#(authorize)\`" below.
325166
+ Givens are also the substrate for access control - see "Access Control: \`#(authorize)\` and \`#(access_filter)\`" below.
324465
325167
 
324466
325168
  ## Legacy: Parameterizable Filters with \`#(filter)\`
324467
325169
 
@@ -324532,38 +325234,72 @@ Publisher formats values based on the dimension's data type, \`string\` → \`'v
324532
325234
 
324533
325235
  Pass \`bypass_filters=true\` (REST) or \`bypassFilters: true\` (POST body) to skip filter injection entirely. Use sparingly, required-filter governance only works if bypass is restricted to trusted callers.
324534
325236
 
324535
- ## Access Control: Source Gating with \`#(authorize)\`
325237
+ ## Access Control: \`#(authorize)\` and \`#(access_filter)\`
325238
+
325239
+ Gate query access to a source over declared \`given:\` values (\`given:\` is Malloy's native runtime-parameter mechanism, the going-forward replacement for \`#(filter)\`). **Two annotations, one question each, and the name tells you which:**
325240
+
325241
+ | Annotation | The question | Body it takes | A denial is |
325242
+ | --- | --- | --- | --- |
325243
+ | \`#(authorize)\` | may this caller reach this source at all? | \`'literal' <op> $GIVEN\`, plus the \`true\`/\`false\` sentinels | **403** |
325244
+ | \`#(access_filter)\` | which rows may they see, once they may? | \`field_path <op> $GIVEN\` | **200**, with their rows |
324536
325245
 
324537
- Gate query access to a source with \`#(authorize)\` over declared \`given:\` values (\`given:\` is Malloy's native runtime-parameter mechanism, the going-forward replacement for \`#(filter)\`). A gate is an \`#(authorize)\` annotation on its own line directly above the \`source:\` line, carrying an **unquoted, ordinary Malloy boolean expression**; Publisher grafts that expression onto the source as a row filter before running the query, so a caller it admits nowhere gets **200 with zero rows**, not a 403. A **403** means only that the gate could not be attached at all. A source with no \`#(authorize)\` annotation of its own or inherited is unrestricted.
325246
+ Either is an annotation on its own line directly above the \`source:\` line, carrying a **narrow grammar publisher parses itself, not an arbitrary Malloy expression**: one or more terms joined only by \`and\`, with \`<op>\` fixed by the given's declared arity (\`in\` for a list, \`=\` for a scalar).
325247
+
325248
+ **Nothing is inferred from the body.** The annotation you write declares which question you are answering, and a body that does not fit is refused at load naming the other annotation. A source with no annotation of either kind, own or inherited, is unrestricted.
325249
+
325250
+ The lock is DECIDED, before the caller's query compiles: a caller it does not admit gets a 403, never a row count or a \`NULL\` aggregate over data they were refused. The filter is GRAFTED onto the query as a \`where:\`, so a caller it matches nowhere gets a 200 with an empty result. The lock runs first; a caller it refuses never reaches the filter. A 403 also covers either gate failing to apply at all (a field the entry point dropped, a given nobody supplied).
324538
325251
 
324539
325252
  \`\`\`malloy
324540
325253
  ##! experimental.givens
324541
325254
 
324542
325255
  given:
324543
325256
  ROLE :: string
325257
+ ORG_IDS :: string[]
324544
325258
 
324545
- #(authorize) $ROLE = 'analyst'
325259
+ #(authorize) 'analyst' = $ROLE
325260
+ #(access_filter) org_id in $ORG_IDS
324546
325261
  source: orders is duckdb.table('orders.parquet') extend {
324547
325262
  measure: order_count is count()
324548
325263
  }
324549
325264
  \`\`\`
324550
325265
 
324551
- - **Any legal Malloy boolean expression is a legal gate**, over givens, row fields (including through a join), literals, functions and operators: \`org_id in $GROUPS\`, \`upper(region) = $REGION\`, \`\` \`cost center\` in $GROUPS \`\`, \`(org_id in $GROUPS or region = $REGION) and amount > $AMOUNTMIN\`. There is no allowlist of accepted comparison shapes.
324552
- - **A source may declare at most one \`#(authorize)\` annotation.** Declaring a second on the same source fails the load naming both. Spell OR inside the expression rather than stacking annotations. For a condition too long to read on one line, point the gate at an ordinary boolean dimension instead: \`#(authorize) authorized\` above the source, over \`dimension: authorized is org_id in $GROUPS\` inside it; validation follows the reference through.
324553
- - **\`#(authorize)\` only gates from the \`source:\` line.** The same annotation on a \`dimension:\`/\`measure:\`/\`join_*:\`/\`view:\` line, or on a top-level \`query:\`, is refused at load naming the position rather than silently protecting nothing.
324554
- - **Every given the gate references must be declared on the entry model's own surface, and must carry no default.** A given the model cannot resolve is refused at load. So is a referenced given declared *with* a default: a caller who supplies nothing would get that default and be admitted or excluded by a value the gate's own line never shows, so it is refused rather than reasoned about case by case. This follows a bare reference through, so a given reached one hop away via \`#(authorize) authorized\` is checked too.
324555
- - **Two shapes load with a warning rather than a refusal.** A gate that references **no given** at all is a fixed predicate, not an access rule keyed on the caller. A gate that **negates a membership test** (\`not (org_id in $GROUPS)\`) matches every row for an *empty* given instead of none. Both warn and still load, so read the load warnings.
324556
- - **Entry point only: not joined, but inherited through \`extend\`.** The gate applies to the source a query enters through. A gate on a source reached only via \`join_*\` **never fires**, at any depth, so anything ungated that joins a locked base hands the base's rows to every caller. A source that \`extend\`s a locked base and declares no gate of its own **does** carry the base's gate; declaring its own annotation replaces it. A source derived from a locked base via a query (\`source: z is locked -> { … }\`) instead **always carries the base's gate in addition to its own**: the derivation recurses into the base unconditionally, so an own gate does not replace it, and the two combine as separate AND'd entries. Pair a locked base (\`#(authorize) false\`) with curated extension sources, using access modifiers (\`include { public: …, private: * }\`), so an extension re-exposes only a curated column surface, and keep sensitive sources out of ungated joins.
324557
- - **A derivation that drops a column the gate reads fails CLOSED.** \`extend { except: org_id }\`, or an \`accept:\` that omits it, leaves the grafted filter unable to compile, so the request is denied rather than served ungated. The one hole to know: dropping the gated column and then \`rename:\`-ing a *different* column onto that exact name grafts successfully and binds the gate to the wrong column. Narrow, but real, so don't recycle a gated column's name.
324558
- - Comparing a row field to an array-typed given with \`=\`/\`!=\` (\`org_id = $GROUPS\`) compiles and loads cleanly, then fails at query execution with a warehouse conversion error. Use \`in\` for an array-typed given, not \`=\`.
324559
- - **The quoted-string and file-level forms are refused at load and no longer exist.** \`#(authorize) "<expr>"\` on the \`source:\` line, in either quote (\`'...'\` is refused the same way), a file-level \`##(authorize) "<expr>"\` applying to every source in the file, and the earlier \`internal dimension: authorized is <expr>\` form are all retired; the load names the rewrite. Every \`.malloy\` file in a package compiles at load and any failure aborts the package, so a retired-form gate anywhere in the package is refused. Only a declaring file *outside* the package escapes that: it loads and denies every request instead, with no compile-time hint. See your deployment's reference documentation.
325266
+ - **Nothing outside the two term shapes above parses.** No \`or\`, \`not\`, \`!=\`, \`<\`/\`>\`/\`<=\`/\`>=\`, function calls, bare field/boolean references, or a literal on the right of a row-level term. \`org_id in $GROUPS\` and \`region = $REGION and org_id in $GROUPS\` are legal; \`upper(region) = $REGION\`, \`(org_id in $GROUPS or region = $REGION)\`, and a bare \`#(access_filter) authorized\` are all refused at load with a named cause (see your deployment's reference documentation for the full list).
325267
+ - **Writing a term on the wrong annotation is refused, both ways.** \`#(authorize) org_id in $GROUPS\` is refused naming \`#(access_filter)\`; \`#(access_filter) 'finance' in $GROUPS\` is refused naming \`#(authorize)\`. The second matters: a constant predicate grafted as a row filter would serve a refused caller 200 with zero rows, which is the answer the lock exists to replace.
325268
+ - **A source may declare more than one note on a route: repeats AND together.** \`#(access_filter) region = $REGION\` stacked with a second \`#(access_filter) org_id in $GROUPS\` both apply, and a caller must satisfy every term across every note. \`or\` is still refused wherever it appears, so there is no way to spell "admit if either" inside one gate or across a source's gates; use two extension sources instead, one per admitted population (see the admin pattern below).
325269
+ - **Two exceptions to "every term references a given", and both live on the lock.** \`#(authorize) false\` is a deny-all: that is how you lock a base. \`#(authorize) true\` is an admit-all, and it is what an extension of a locked base needs in order to be open: a source declaring no gate of its own **inherits** its ancestor's, so leaving the annotation off an extension of a \`false\` base inherits the lock rather than lifting it. Neither sentinel is legal on \`#(access_filter)\`. \`false\` may not share a source with another note at all; \`true\` may not share the lock route with one, but is live beside an \`#(access_filter)\` (\`#(authorize) true\` with \`#(access_filter) org_id in $GROUPS\` re-opens a locked base while still scoping the rows). Neither is a term: \`true and org_id in $GROUPS\` is read as an ordinary two-term body whose first term is malformed.
325270
+ - **Inheritance is per route.** Both inherit through \`extend\` the same way, own wins over ancestor on that route only, so a source can be "own" for one and "inherited" for the other. The API reports them separately, each under the field named for its own annotation: \`Source.authorize\` and \`Source.accessFilter\`.
325271
+ - **Generate the admin escape hatch as another extension over the locked base, never as a bypass.** When a model needs an ordinary population plus a wider one, add a second extension whose rule is a lock on the caller:
325272
+
325273
+ \`\`\`malloy
325274
+ given:
325275
+ ORG_IDS :: string[]
325276
+ GROUPS :: string[]
325277
+
325278
+ #(authorize) false
325279
+ source: orders_base is duckdb.table('orders.parquet') extend {}
325280
+
325281
+ #(authorize) true
325282
+ #(access_filter) org_id in $ORG_IDS
325283
+ source: orders is orders_base extend {}
325284
+
325285
+ #(authorize) 'admin' in $GROUPS
325286
+ source: orders_admin is orders_base extend {}
325287
+ \`\`\`
325288
+
325289
+ Each extension replaces the base's \`false\` on the lock route with its own rule. \`orders\` is open to everyone and scoped per row; \`orders_admin\` is locked to the admin group and, carrying no filter, serves every row to whoever passes. This is the shape to generate whenever an author wants a role to see everything: another extension source, not a flag that skips the gate.
325290
+ - **Both only gate from the \`source:\` line.** The same annotation on a \`dimension:\`/\`measure:\`/\`join_*:\`/\`view:\` line, or on a top-level \`query:\`, is refused at load naming the position rather than silently protecting nothing.
325291
+ - **Every given the gate references must be declared on the entry model's own surface, and must carry no default.** A given the model cannot resolve is refused at load. So is a referenced given declared *with* a default: a caller who supplies nothing would get that default and be admitted or excluded by a value the gate's own line never shows, so it is refused rather than reasoned about case by case.
325292
+ - **A scalar/array mismatch between the operator and the given's declared type is a load-time refusal**, not a request-time warehouse error: \`org_id in $GROUPS\` requires \`GROUPS\` to be array-typed, \`region = $REGION\` requires \`REGION\` scalar. Negation is likewise refused outright, so there is no empty-given inversion surprise to warn about.
325293
+ - **Entry point only: not joined, but inherited through \`extend\`.** The gate applies to the source a query enters through. A gate on a source reached only via \`join_*\` **never fires**, at any depth, so anything ungated that joins a locked base hands the base's rows to every caller. A source that \`extend\`s a locked base and declares no gate of its own **does** carry the base's gate; declaring its own annotation replaces it. A source derived from a locked base via a query (\`source: z is locked -> { … }\`) instead **always carries the base's gate in addition to its own**: the derivation recurses into the base unconditionally, so an own gate does not replace it, and the two combine as separate AND'd entries. Pair a locked base with curated extension sources, using access modifiers (\`include { public: …, private: * }\`), so an extension re-exposes only a curated column surface, and keep sensitive sources out of ungated joins.
325294
+ - **A derivation that drops a column the gate reads fails CLOSED.** \`extend { except: org_id }\`, or an \`accept:\` that omits it, leaves the grafted filter unable to compile, so the request is denied rather than served ungated. The one hole to know: dropping the gated column and then \`rename:\`-ing a *different* column onto that exact name grafts successfully and binds the gate to the wrong column. Narrow, but real, so don't recycle a gated column's name. If a derivation's own projection needs to drop the column a row-level term would read, use a source-level term (\`'literal' in/= $GIVEN\`) instead, since it never depends on any projected column.
325295
+ - **The quoted-string and file-level forms are refused at load and no longer exist.** A quoted expression on the \`source:\` line, in either quote, a file-level \`##(…) "<expr>"\` applying to every source in the file, and the earlier \`internal dimension: authorized is <expr>\` form are all retired; the load names the rewrite. Every \`.malloy\` file in a package compiles at load and any failure aborts the package, so a retired-form gate anywhere in the package is refused. Only a declaring file *outside* the package escapes that: it loads and denies every request instead, with no compile-time hint. See your deployment's reference documentation.
324560
325296
  - **A gated source can be persisted, but the gating column freezes.** \`storage=\` and \`#@ preaggregate\` refuse a gated source outright; a colocated \`#@ persist\` is admitted when the gate is provably the entry point's own row filter. The gate still runs live on every query, so rows come back filtered - but the column it filters ON is frozen at build time, so a row whose access decision changes keeps being served under the old decision until the next rebuild. Pair \`#@ persist\` on a gated source with a freshness window (\`fallback="live"\`), which is the only control that bounds that - and read \`skill:malloy-materialization\` for where that window binds, because on a standalone Publisher it does not.
324561
325297
 
324562
- > **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`. \`#(authorize)\` is only a real boundary when it sits behind a trusted tier that sets givens from its own verified context, never directly from an untrusted caller. It is not, on its own, end-user authentication.
325298
+ > **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`. Neither annotation is a real boundary unless it sits behind a trusted tier that sets givens from its own verified context, never directly from an untrusted caller. Neither is, on its own, end-user authentication.
324563
325299
  >
324564
- > **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning \`#(authorize)\` into a standalone boundary. Model access on \`given:\` + \`#(authorize)\` now; it is the surface that carries forward.
325300
+ > **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning these gates into a standalone boundary. Model access on \`given:\` + these two annotations now; it is the surface that carries forward.
324565
325301
 
324566
- Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's \`#(authorize)\` reference documentation.
325302
+ Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's authorize reference documentation.
324567
325303
 
324568
325304
  ## Join Syntax
324569
325305
 
@@ -325192,7 +325928,7 @@ source: first_touch is conn.table('events') -> {
325192
325928
  {
325193
325929
  name: "malloy-modeling",
325194
325930
  description: "Build semantic models with Malloy for the Malloy Publisher. Read this skill whenever the user asks about modeling data or specifically mentions Malloy.",
325195
- body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself before writing ANY code, with the tool that matches what you are modelling.\n - Modelling data **already in a package**: `get_context` returns that package's sources, views, and fields (with their docs).\n - Modelling **a database with no package yet**: `get_context` has nothing to return, so use `search_database_schema` instead. It walks the connection's schemas and tables, ranks them against a plain-English description, and gives you each table's columns plus the `source:` line to start from. Take those names verbatim into step 5.\n Never guess field names either way.\n2. **Search docs proactively**: call `search_malloy_docs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **`count()` counts rows; `count(x)` counts distinct values of `x`**: `count(distinct x)` is deprecated, write `count(x)`\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files.\n\n`modeling-notes.md` is an expected output of the workflow, not an optional extra. Start it at step 2 (Propose Scope) and grow it as you work: it persists alongside the model, and its value is as the thing the user argues with at step 3, before source files exist; written after the build it can only document decisions already baked in. Record findings and problems as they are found during discovery (`skill:malloy-discover`), and every unconfirmed decision as an open item. Only when there is no writable workspace do the notes live in the conversation instead.\n\nKeep it compact, with these sections:\n\n```markdown\n# Modeling notes - <package>\n## Scope what was confirmed, what the model is FOR, skip list with reasons\n## Grain and keys proven by query, not by column name\n## Coverage coverage cliffs; columns excluded for nullity\n## Decisions each with its evidence\n## Open decisions ASSUMPTIONS, NOT CONFIRMED: every threshold or definition the user\n has not settled, one entry each, mirrored by a hedge in its #(doc)\n## Validation reconciliation checks performed, and their results\n```\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions. Read each step's skill **before starting that step**, including the decision skills for steps 1–4 (`skill:malloy-discover`, `skill:malloy-scope`, `skill:malloy-define`). They govern what the model says; skipping them to reach the build skills is how unreviewed business logic ships.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags, and any threshold or business convention in it is user-confirmed, distribution-derived, or explicitly flagged in its `#(doc)`** (see `skill:malloy-document` § Mark conventions as conventions). Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent) (user) (user) (user) (agent) (agent) (user) (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art. With no package yet, start from `search_database_schema` to find the tables in the connection |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present the review checklist below; user confirms or corrects |\n| 8. Curate | `skill:malloy-model` | Propose access controls (`explores`, `queryableSources`, access modifiers); always propose, the user decides whether to apply |\n\n### The pauses are the point\n\nThese are governed semantic models: the business decisions in them must be confirmed by a human subject-matter expert, and the **(user)** steps exist to collect that confirmation. They are real stops, not progress reports. A model can be complete, compiling, and fully documented and still be wrong everywhere it guessed; a capable agent can build the whole thing without pausing once, which is exactly the failure mode this workflow exists to prevent.\n\nWhen a decision goes unanswered (the user explicitly declines to decide, or nobody is there to ask), do not silently proceed as if it were settled. Take your best-supported position, label it an assumption in the field's own `#(doc)` (see `skill:malloy-document` § Mark conventions as conventions), record it under \"Open decisions\" in `modeling-notes.md`, and raise it again at Review. An unlabeled assumption is indistinguishable from a confirmed fact, and misleads everyone downstream.\n\n### Step 7 Review is a checklist, not a summary\n\nPresent these to the user, with answers:\n\n- **Which definitions did the user actually confirm?** List them; everything else is an assumption.\n- **Which thresholds and bucket boundaries did you choose?** For each: the evidence (distribution query, metadata, prior art) and the `#(doc)` hedge that marks it.\n- **Which questions were left unanswered?** Each must already carry a labeled assumption and an \"Open decisions\" entry.\n- **Does the headline metric have more than one defensible definition?** If yes, that is a blocking question: put the candidate definitions to the user with their counts side by side, not in a footnote.\n\nThe user confirming this checklist is what makes the model governed. A summary of what you built is not a checkpoint.\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** a data question arrives before any model exists → `skill:malloy-model-as-you-go`. It answers the question with `skill:malloy-analysis`, then codifies what the answer assumed into the model, one question at a time, confirming binding decisions first. The model exists by the end; there is no separate formalize step.\n- **Open-ended exploration** with no intent to keep anything: `skill:malloy-analyze`. If it turns into something worth keeping, formalize via `skill:malloy-model` (`reference/analysis-to-model.md`).\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through Review and propose Curate. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-model-as-you-go`: answer their first real question, codify what it assumed, repeat |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-model-as-you-go`. It answers with `skill:malloy-analysis` and grows the model from what each answer assumed, so there is nothing to formalize afterwards.\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling. No server yet? `skill:malloy-getting-started` covers setup, including the one-command scaffolder (`npm create @malloy-publisher/malloy-package@latest <name>`) and why local authoring needs `--watch-env <env>`: start the server without it and your saved edits are never read.\n\n| Tool | Purpose |\n|------|---------|\n| `get_context` | Ground yourself in a package: its sources, views, and fields |\n| `execute_query` | Run ad-hoc queries for validation |\n| `compile_model` | Compile-check a change and get diagnostics back without running a query |\n| `reload_package` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `search_malloy_docs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n| `search_database_schema` | Find the tables in a database connection by plain-English description, when modelling data that is not in a package yet. Returns each table's columns and the `source:` line to start from. Names and types only: no row value is returned |\n\nNever guess field names. Ground yourself with `get_context` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `compile_model`, picking the scope that matches what you are doing:\n - Adding a new definition or query: the default (`scope: \"append\"`) compiles your text in the model's namespace. Note its diagnostic positions land in the model-plus-your-text concatenation.\n - **Editing an existing definition: `scope: \"file\"`**, with the whole edited file as `source`. It compiles your text AS the file (append would collide with \"Cannot redefine\"), and diagnostics land at the true line numbers of your text.\n - Before saving a change other files import: `scope: \"package\"` with the edited file as `source` runs reload's worker compiler over every `.malloy` and `.malloynb` file against your edit, so a rename that breaks an importer surfaces now instead of at reload. Each diagnostic carries `model`, the file it points at; files hidden from discovery can appear. If `modelPath` does not exactly match an existing file, a warning says the source was treated as new.\n2. **Save** it to the package's model file.\n3. **Reload** with `reload_package`.\n4. **Run** the new view with `execute_query`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback, and a `scope: \"package\"` dry-run with no `source` uses reload's compiler and file selection (imports across files, every `.malloy` and `.malloynb` file as saved) without touching the served model. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `execute_query`. An older Publisher that has `compile_model` but rejects `scope` supports only the append behavior.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ... RIGHT: source: flights is ...\nWRONG: dimension: x as y RIGHT: dimension: y is x\nWRONG: count(*) RIGHT: count()\nWRONG: count(distinct x) RIGHT: count(x)\nWRONG: revenue / order_count RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... } RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns"
325931
+ body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself before writing ANY code, with the tool that matches what you are modelling.\n - Modelling data **already in a package**: `get_context` returns that package's sources, views, and fields (with their docs).\n - Modelling **a database with no package yet**: `get_context` has nothing to return, so use `search_database_schema` instead. It walks the connection's schemas and tables, ranks them against a plain-English description, and gives you each table's columns plus the `source:` line to start from. Take those names verbatim into step 5.\n Never guess field names either way.\n2. **Search docs proactively**: call `search_malloy_docs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **`count()` counts rows; `count(x)` counts distinct values of `x`**: `count(distinct x)` is deprecated, write `count(x)`\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files.\n\n`modeling-notes.md` is an expected output of the workflow, not an optional extra. Start it at step 2 (Propose Scope) and grow it as you work: it persists alongside the model, and its value is as the thing the user argues with at step 3, before source files exist; written after the build it can only document decisions already baked in. Record findings and problems as they are found during discovery (`skill:malloy-discover`), and every unconfirmed decision as an open item. Only when there is no writable workspace do the notes live in the conversation instead.\n\nKeep it compact, with these sections:\n\n```markdown\n# Modeling notes - <package>\n## Scope what was confirmed, what the model is FOR, skip list with reasons\n## Grain and keys proven by query, not by column name\n## Coverage coverage cliffs; columns excluded for nullity\n## Decisions each with its evidence\n## Open decisions ASSUMPTIONS, NOT CONFIRMED: every threshold or definition the user\n has not settled, one entry each, mirrored by a hedge in its #(doc)\n## Validation reconciliation checks performed, and their results\n```\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions. Read each step's skill **before starting that step**, including the decision skills for steps 1–4 (`skill:malloy-discover`, `skill:malloy-scope`, `skill:malloy-define`). They govern what the model says; skipping them to reach the build skills is how unreviewed business logic ships.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags, and any threshold or business convention in it is user-confirmed, distribution-derived, or explicitly flagged in its `#(doc)`** (see `skill:malloy-document` § Mark conventions as conventions). Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent) (user) (user) (user) (agent) (agent) (user) (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art. With no package yet, start from `search_database_schema` to find the tables in the connection |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present the review checklist below; user confirms or corrects |\n| 8. Curate | `skill:malloy-model` | Propose the published surface (an `index.malloy` with `export { ... }`) and access controls (access modifiers, gates); always propose, the user decides whether to apply |\n\n### The pauses are the point\n\nThese are governed semantic models: the business decisions in them must be confirmed by a human subject-matter expert, and the **(user)** steps exist to collect that confirmation. They are real stops, not progress reports. A model can be complete, compiling, and fully documented and still be wrong everywhere it guessed; a capable agent can build the whole thing without pausing once, which is exactly the failure mode this workflow exists to prevent.\n\nWhen a decision goes unanswered (the user explicitly declines to decide, or nobody is there to ask), do not silently proceed as if it were settled. Take your best-supported position, label it an assumption in the field's own `#(doc)` (see `skill:malloy-document` § Mark conventions as conventions), record it under \"Open decisions\" in `modeling-notes.md`, and raise it again at Review. An unlabeled assumption is indistinguishable from a confirmed fact, and misleads everyone downstream.\n\n### Step 7 Review is a checklist, not a summary\n\nPresent these to the user, with answers:\n\n- **Which definitions did the user actually confirm?** List them; everything else is an assumption.\n- **Which thresholds and bucket boundaries did you choose?** For each: the evidence (distribution query, metadata, prior art) and the `#(doc)` hedge that marks it.\n- **Which questions were left unanswered?** Each must already carry a labeled assumption and an \"Open decisions\" entry.\n- **Does the headline metric have more than one defensible definition?** If yes, that is a blocking question: put the candidate definitions to the user with their counts side by side, not in a footnote.\n\nThe user confirming this checklist is what makes the model governed. A summary of what you built is not a checkpoint.\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** a data question arrives before any model exists → `skill:malloy-model-as-you-go`. It answers the question with `skill:malloy-analysis`, then codifies what the answer assumed into the model, one question at a time, confirming binding decisions first. The model exists by the end; there is no separate formalize step.\n- **Open-ended exploration** with no intent to keep anything: `skill:malloy-analyze`. If it turns into something worth keeping, formalize via `skill:malloy-model` (`reference/analysis-to-model.md`).\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through Review and propose Curate. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-model-as-you-go`: answer their first real question, codify what it assumed, repeat |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-model-as-you-go`. It answers with `skill:malloy-analysis` and grows the model from what each answer assumed, so there is nothing to formalize afterwards.\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling. No server yet? `skill:malloy-getting-started` covers setup, including the one-command scaffolder (`npm create @malloy-publisher/malloy-package@latest <name>`) and why local authoring needs `--watch-env <env>`: start the server without it and your saved edits are never read.\n\n| Tool | Purpose |\n|------|---------|\n| `get_context` | Ground yourself in a package: its sources, views, and fields |\n| `execute_query` | Run ad-hoc queries for validation |\n| `compile_model` | Compile-check a change and get diagnostics back without running a query |\n| `reload_package` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `search_malloy_docs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n| `search_database_schema` | Find the tables in a database connection by plain-English description, when modelling data that is not in a package yet. Returns each table's columns and the `source:` line to start from. Names and types only: no row value is returned |\n\nNever guess field names. Ground yourself with `get_context` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `compile_model`, picking the scope that matches what you are doing:\n - Adding a new definition or query: the default (`scope: \"append\"`) compiles your text in the model's namespace. Note its diagnostic positions land in the model-plus-your-text concatenation.\n - **Editing an existing definition: `scope: \"file\"`**, with the whole edited file as `source`. It compiles your text AS the file (append would collide with \"Cannot redefine\"), and diagnostics land at the true line numbers of your text.\n - Before saving a change other files import: `scope: \"package\"` with the edited file as `source` runs reload's worker compiler over every `.malloy` and `.malloynb` file against your edit, so a rename that breaks an importer surfaces now instead of at reload. Each diagnostic carries `model`, the file it points at; files hidden from discovery can appear. If `modelPath` does not exactly match an existing file, a warning says the source was treated as new.\n2. **Save** it to the package's model file.\n3. **Reload** with `reload_package`.\n4. **Run** the new view with `execute_query`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback, and a `scope: \"package\"` dry-run with no `source` uses reload's compiler and file selection (imports across files, every `.malloy` and `.malloynb` file as saved) without touching the served model. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `execute_query`. An older Publisher that has `compile_model` but rejects `scope` supports only the append behavior.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ... RIGHT: source: flights is ...\nWRONG: dimension: x as y RIGHT: dimension: y is x\nWRONG: count(*) RIGHT: count()\nWRONG: count(distinct x) RIGHT: count(x)\nWRONG: revenue / order_count RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... } RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns"
325196
325932
  },
325197
325933
  {
325198
325934
  name: "malloy-notebook-chat",
@@ -325216,7 +325952,7 @@ Steps to follow when the user asks a question:
325216
325952
  {
325217
325953
  name: "malloy-notebooks",
325218
325954
  description: 'Create Malloy notebooks (.malloynb) for interactive dashboards and data stories. Use when user asks to "create a notebook", "make a dashboard notebook", "write a malloynb", "data story", or needs to build reports/visualizations.',
325219
- body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
325955
+ body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)`/`#(access_filter)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
325220
325956
  },
325221
325957
  {
325222
325958
  name: "malloy-patterns",
@@ -325226,7 +325962,7 @@ Steps to follow when the user asks a question:
325226
325962
  {
325227
325963
  name: "malloy-phrase-detection",
325228
325964
  description: "How to phrase search_text on a get_context call so retrieval returns the fields you need instead of a truncated catalog. Covers target-type classification and decomposition patterns.",
325229
- body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), not the distinct categorical *values* stored in the data. To find which literal values a categorical dimension holds, target the dimension, then query its distinct values with `execute_query` (see the patterns below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values (no value-search target in v1).** When the user names a literal value like \"premium\" or \"New York City\", target the *dimension* it lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`). Then confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it. The data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value by querying the status dimension's distinct values with `execute_query`.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with `execute_query`.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; enumerate the exact stored values with `execute_query`.\n8. **The quantity asked for is a measure, even when nothing names it.** \"How many titles were released in 2019?\" is asking for a count, but no noun in it is the count: the visible phrases are the subject (\"titles\") and the filter (\"2019\"), and mapping only those yields a `source` and a `dimension` target with no way to answer. Always add a measure target for the quantity itself (`\"the number of titles\"`). This holds for every interrogative that IS the aggregation -- \"how many\", \"how much\", \"how often\" -- and it is the pattern most easily lost when a filter or grouping is the loud part of the sentence. Measured: an answerer got this right on \"How many titles are in the dataset?\" and dropped it on \"How many titles were released in 2019?\", where the year took the attention.\n9. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" do not get their own value targets, they resolve to the city and tier dimensions. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then run `execute_query` to read the city and tier dimensions' distinct values and confirm the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\").\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
325965
+ body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` / `dimensional_value` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), and `dimensional_value` targets search the categorical *values* stored in the data. Some servers don't index values; check the tool's description. On one that doesn't, find a literal value by targeting its dimension, then querying its distinct values with `execute_query` (see \"Where value search isn't available\" below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`dimensional_value`**: a literal value the user named, stored in some dimension. `search_text` is the value itself, the one target where echoing the user's word is right.\n - \"CyberArk\" stays `\"CyberArk\"`\n - Best practice is to scope the call to the source that holds the value. An unscoped value search covers every indexed dimension in scope, so it is slow on a large package. If you truly don't know the source, you can leave the scope off, but expect a slow call.\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values.** When the user names a literal value like \"premium\" or \"New York City\", send a `dimensional_value` target for it, scoped to its source once you know the source. Filter on the exact string it returns: the data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n**Where value search isn't available.** Some servers have no value index and return nothing for a `dimensional_value` target, and on others a particular dimension's values may not be indexed. Then target the *dimension* the value lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`), and confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value with a `dimensional_value` target, or the status dimension's distinct values where values aren't indexed.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with a `dimensional_value` target, or `execute_query` where values aren't indexed.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; confirm the exact stored values with `dimensional_value` targets, or `execute_query` where values aren't indexed.\n8. **The quantity asked for is a measure, even when nothing names it.** \"How many titles were released in 2019?\" is asking for a count, but no noun in it is the count: the visible phrases are the subject (\"titles\") and the filter (\"2019\"), and mapping only those yields a `source` and a `dimension` target with no way to answer. Always add a measure target for the quantity itself (`\"the number of titles\"`). This holds for every interrogative that IS the aggregation -- \"how many\", \"how much\", \"how often\" -- and it is the pattern most easily lost when a filter or grouping is the loud part of the sentence. Measured: an answerer got this right on \"How many titles are in the dataset?\" and dropped it on \"How many titles were released in 2019?\", where the year took the attention.\n9. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" resolve to the city and tier dimensions in this first call, because the source is not known yet and an unscoped value search is slow. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then send a second call scoped to `subscriptions`, with `dimensional_value` targets for `\"NYC\"`, `\"premium\"` and `\"basic\"`, to get the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\"). Where values aren't indexed, run `execute_query` on the city and tier dimensions' distinct values instead.\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
325230
325966
  },
325231
325967
  {
325232
325968
  name: "malloy-publish",
@@ -325279,28 +326015,47 @@ If it doesn't exist, create one. Suggest a package name based on the model conte
325279
326015
 
325280
326016
  ### Curating discovery & the query boundary (optional)
325281
326017
 
325282
- By default a package exposes **everything**: every model is listed and every source is directly queryable. That's the right behavior for most packages, and it's the safe default: **omit these fields and nothing changes.** Reach for them when you have raw/staging/scaffolding sources that exist to build a curated entry point and you don't want agents landing on (or querying) them directly.
326018
+ A package with no \`index.malloy\` and no \`explores\` exposes **everything**: every model is listed and every source is directly queryable.
326019
+
326020
+ To curate, add an **\`index.malloy\`** at the package root. Publisher reads it as the package's published surface, so no manifest field is involved:
326021
+
326022
+ \`\`\`malloy
326023
+ // index.malloy
326024
+ import "order_analysis.malloy"
326025
+ import "staging.malloy"
326026
+
326027
+ export { orders, customers }
326028
+ \`\`\`
326029
+
326030
+ What it exports is what agents discover **and** what may be queried. Everything else still compiles, and other models can import, join and extend it, but a direct query against it is refused with a 404 - indistinguishable from a source that does not exist. Reach for this when you have raw/staging/scaffolding sources that exist to build a curated entry point and you don't want agents landing on, or querying, them directly.
326031
+
326032
+ **Address queries to the surface.** Once a package has an \`index.malloy\`, \`.../models/staging.malloy/query\` is no longer a query entry point, *even for a source that file declares itself*. Use \`.../models/index.malloy/query\`. If you are debugging a refusal rather than authoring, \`skill:malloy-source-unreachable\` covers the three ways a source can be out of reach and how to tell them apart.
326033
+
326034
+ **A surface can be layered.** An \`index.malloy\` may front a file that fronts another. A source re-exported through a chain of files stays queryable through the surface at any depth, because admission follows the declaration rather than the path taken to it.
326035
+
326036
+ **Curation hides a landing point, not a column.** A published source may \`join\` an unpublished one, and a query grouping by a joined field returns that field's values normally. If a column must not be readable, do not join it into something you publish; gate it with \`#(authorize)\` instead.
325283
326037
 
325284
- Two optional fields opt the package into curation:
326038
+ **Leaving a source out does not put it out of reach, but there is a condition.** \`export { ... }\` also decides what an *importing* file may see, which is Malloy's rule rather than Publisher's. A file that declares no \`export\` hands an importer everything it declares, so an unpublished source stays importable and joinable from the file that declares it. A file that *does* declare one hands over exactly that list: importing a file whose \`export\` omits a source and then naming it fails to compile with \`Reference to undefined object\`. Put an \`export\` on a mid-layer file only when you mean to narrow what its importers can build on, not just what Publisher lists.
326039
+
326040
+ **About \`export { … }\`:** the surface filters which *files* are listed; \`export { … }\` (a Malloy statement) filters which *sources within a file* are exposed, and the two compose. You usually don't write it in a leaf model: a file with **no** \`export\` exposes all of its own top-level sources. It must appear after the definitions it names. See [Malloy: Imports & Exports](https://docs.malloydata.dev/documentation/language/imports).
326041
+
326042
+ ### The older manifest fields
326043
+
326044
+ Both still work and are not going away in this release. Both are deprecated where \`index.malloy\` replaces them, and a package using them that way gets a load-time warning naming the replacement. The two uses it cannot replace, an \`explores\` naming several files and \`queryableSources: "all"\`, stay supported without a warning. Do not add either key to a new package that does not need one of those two.
325285
326045
 
325286
326046
  \`\`\`json
325287
326047
  {
325288
326048
  "name": "ecommerce",
325289
- "version": "0.0.1",
325290
- "description": "Orders, customers, and revenue analysis",
325291
326049
  "explores": ["order_analysis.malloy", "customer_health.malloy"],
325292
- "queryableSources": "declared"
326050
+ "queryableSources": "all"
325293
326051
  }
325294
326052
  \`\`\`
325295
326053
 
325296
- - **\`explores\`** (\`string[]\`) - an allowlist of **model file paths** (relative to the package root, not source or view names) whose models agents should discover and land on. Declaring it is the single opt-in for all discovery curation. With \`explores\` set, listings narrow to those files, and within each file to its \`export { ... }\` closure (below), so anything a listed file doesn't export is also dropped. Other files still compile, and can still be imported or joined, but are hidden from listings. Leaving \`explores\` **absent or empty** means every model is listed, unchanged from today, so existing packages don't break when this field is added. An entry that doesn't resolve to a real \`.malloy\` file surfaces in \`exploresWarnings\`; publishing a package that has any is rejected, so fix the path before publishing.
325297
- - **\`queryableSources\`** (\`"declared"\` | \`"all"\`, default \`"declared"\`) - the query boundary. Only takes effect once \`explores\` is set. \`"declared"\` makes queryable == discoverable: only the \`explores\` files and their \`export {}\` closure are valid top-level query targets; other sources still compile, import, and join, but a direct query against one is denied. \`"all"\` curates discovery only: every compiled source stays queryable even though \`explores\` narrows what's listed.
325298
-
325299
- **About \`export { … }\`:** \`explores\` filters which *files* are listed; \`export { … }\` (a Malloy statement) filters which *sources within a file* are exposed, the two compose. You usually don't write it: a file with **no** \`export\` exposes all of its own top-level sources. Add \`export { orders, customers }\` to a file to expose only those and keep imported/scaffolding helpers out of discovery (it must appear after the definitions it names). See [Malloy: Imports & Exports](https://docs.malloydata.dev/documentation/language/imports).
326054
+ - **\`explores\`** (\`string[]\`) - model file paths relative to the package root, naming the surface. Reach for it for the one thing \`index.malloy\` cannot express: a surface spanning **several** files. An explicit \`explores\` always wins over the convention, and a package with both an \`index.malloy\` and an \`explores\` that omits it carries a warning saying so. An entry that doesn't resolve to a real \`.malloy\` file surfaces in \`exploresWarnings\`; publishing a package that has any is rejected, so fix the path before publishing. An explicit \`"explores": []\` means "do not curate" and suppresses the convention.
326055
+ - **\`queryableSources\`** (\`"declared"\` | \`"all"\`, default \`"declared"\`) - the query boundary. \`"declared"\` is already the default, so the key changes nothing. **\`"all"\` is the exception, and \`index.malloy\` does not replace it:** it curates listings while leaving every source queryable by name, and a surface derived from an \`index.malloy\` always enforces the boundary. If you want listings-only curation, keep both keys.
325300
326056
 
325301
- **Why curate here:** declaring \`explores\` routes agents to the well-documented curated sources instead of raw tables, and \`queryableSources: "declared"\` keeps them from reaching the hidden sources by name. The two axes compose: list a file in \`explores\` for its models to be discoverable, and \`export\` a source within that file for it to be a landing point.
325302
326057
 
325303
- > **Not access control.** \`queryableSources\` gates the query surface (the query endpoints, REST and MCP alike), not compile and not raw file retrieval by exact path: \`/compile\` and \`compile_model\` are deliberately exempt, because compile is the authoring loop and the boundary is discovery curation. It doesn't restrict *who* may query, only *what* is queryable by name. Queryable sources are the union of every \`explores\`-listed file's \`export {}\` closure, whichever listed model path a query addresses them through. To gate access by caller-supplied identity/role, use \`#(authorize)\` on the source, see \`skill:malloy-model\` § Access Control and \`docs/authorize.md\`. Discovery curation and \`#(authorize)\` are independent layers.
326058
+ > **Not access control.** The surface gates the query surface (the query endpoints, REST and MCP alike), not compile and not raw file retrieval by exact path: \`/compile\` and \`compile_model\` are deliberately exempt, because compile is the authoring loop and the boundary is discovery curation. It doesn't restrict *who* may query, only *what* is queryable by name. Queryable sources are the union of every listed file's \`export {}\` closure, whichever listed model path a query addresses them through. To gate access by caller-supplied identity/role, use \`#(authorize)\` on the source (and \`#(access_filter)\` to scope rows), see \`skill:malloy-model\` § Access Control and \`docs/authorize.md\`. Discovery curation and these gates are independent layers.
325304
326059
 
325305
326060
  The manifest also carries a \`scope\` field (\`"package"\` | \`"version"\`, default \`"package"\`) controlling whether persisted/materialized artifacts are shared across published versions or owned by a single version, and a \`materialization\` field configuring that persistence policy (a cron \`schedule\` or a \`freshness\` window). Both are unrelated to discovery curation; there is no per-source \`sharing\` or \`schedule\` field, that was retired in favor of the single package-level \`scope\` and \`materialization\`.
325306
326061
 
@@ -325433,10 +326188,10 @@ Wrong: \`pick('Premium') { ... }\` (that's not Malloy syntax).
325433
326188
  **Window functions with \`calculate:\`:** running totals, \`lag()\`, \`lead()\`, and other window operations belong in \`calculate:\`, not \`aggregate:\`.
325434
326189
  \`\`\`malloy
325435
326190
  run: source -> {
325436
- group_by: month is order_date.month
326191
+ group_by: order_month is order_date.month
325437
326192
  aggregate: revenue
325438
326193
  calculate: prev_month_revenue is lag(revenue)
325439
- order_by: month
326194
+ order_by: order_month
325440
326195
  }
325441
326196
  \`\`\`
325442
326197
 
@@ -325474,15 +326229,15 @@ where: order_date >= @2020-01-01
325474
326229
 
325475
326230
  ### Truncation accessors return timestamps, not integers
325476
326231
 
325477
- \`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer, and it cannot be chained: writing \`order_date.month.day\` fails. Stop at the first truncation.
326232
+ \`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer.
325478
326233
 
325479
- Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`. Stop at the first.
326234
+ Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`.
325480
326235
 
325481
- ### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved words
326236
+ ### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved as names
325482
326237
 
325483
- Don't try to extract a numeric month with a function call: \`month(order_date)\`, \`year(order_date)\`, etc. fail to parse in \`where:\` because the names are reserved. The same names can also break aliases:
326238
+ They cannot name an output field. The functions are fine: \`month(order_date)\` extracts 1-12 anywhere, including \`where:\`.
325484
326239
 
325485
- Wrong: \`group_by: month is order_date.month\` → parse error
326240
+ Wrong: \`group_by: month is order_date.month\` → \`'month' is a reserved word, so to use it as a name you must quote it\`
325486
326241
  Right: \`group_by: order_month is order_date.month\`
325487
326242
  Right (when a column is literally named \`month\`): \`\` group_by: \`month\` \`\`
325488
326243
 
@@ -325598,7 +326353,7 @@ run: source -> {
325598
326353
  }
325599
326354
  \`\`\`
325600
326355
 
325601
- Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A misplaced chart annotation (e.g., inside \`{ }\`) typically produces the error *"field is a bar chart, but is not a repeated record"*.
326356
+ Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A chart annotation left as the last line inside \`{ }\` fails with *"Parser enountered unexpected statement type 'unimplemented'"* (the compiler's spelling); one after the closing \`}\` fails with *"Object annotation not connected to any object"*.
325602
326357
 
325603
326358
  Read the \`malloy-charts\` skill for chart types, properties, data shape requirements, and selection guidance.
325604
326359
 
@@ -325609,11 +326364,10 @@ Read the error against the tables above and below. Most failures match a known p
325609
326364
  | Error message | Likely cause / fix |
325610
326365
  |---|---|
325611
326366
  | \`Cannot compare a timestamp to a number\` | Comparing \`date.year\` to an integer. Use \`date >= @2020-01-01\` instead. |
325612
- | \`no viable alternative at input '<word>'\` | Two common causes: a reserved keyword used as an alias or as a function call (e.g., \`month is ...\`, \`month(date_field)\`) - rename, backtick, or use \`date_field.month\` / a \`?\`-apply filter instead; or a semicolon used to separate fields within a clause - fields under one \`aggregate:\`/\`group_by:\` are comma- or newline-separated, never \`;\` (the error points at the field right after the \`;\`). |
326367
+ | \`no viable alternative at input '<word>'\` | Often a \`;\` between fields within one clause - fields under one \`aggregate:\`/\`group_by:\` are comma- or newline-separated (the error points at the field right after the \`;\`). |
325613
326368
  | \`'<field_name>' is not defined\` | Field doesn't exist in the source. Re-check against the model definition; you may have stripped a join prefix. |
325614
- | \`missing {DAY, HOUR, MINUTE, MONTH, QUARTER, SECOND, WEEK, YEAR}\` | Chained date property too deep (e.g., \`.month.something\`). Stop at the first truncation. |
325615
326369
  | \`field is a bar chart, but is not a repeated record\` | Chart annotation placed inside \`{ }\`. Move \`# bar_chart\` above \`run:\` / \`view:\` / \`nest:\`. |
325616
- | \`Parser encountered unexpected statement\` | Unsupported feature, or syntax placed where Malloy doesn't allow it (e.g., \`pick\` inside a nested view). |
326370
+ | \`Parser enountered unexpected statement\` | Spelled that way by the compiler. Most often a chart annotation left as the last line inside \`{ }\` - move it above \`run:\`. Also syntax Malloy doesn't allow in that position (e.g., \`pick\` inside a nested view). |
325617
326371
  | Query silently returns zero rows | Filter value mismatch (case, spelling, format). Run a distinct-values query on the dimension to confirm the literal. |
325618
326372
 
325619
326373
  ## Syntax Help
@@ -326304,6 +327058,74 @@ Then hand off to modeling: use your modeling workflow to turn the confirmed scop
326304
327058
  ## Done
326305
327059
 
326306
327060
  Scope confirmed, in the conversation and in \`modeling-notes.md\`: tables in scope, analytical focus, model kind and what it rules out, and what's deferred. Continue with your modeling workflow.`
327061
+ },
327062
+ {
327063
+ name: "malloy-source-unreachable",
327064
+ description: `Work out why a source is missing from get_context, or why a query against it was refused with a 404 or a 403, and what to do next. Use when a source you expect is absent from discovery, when execute_query answers "No queryable source", when a query that used to work stops working, or when deciding whether a package's curation is hiding something you need.`,
327065
+ body: `# A source you cannot reach
327066
+
327067
+ > **Tool names** are written bare here - \`get_context\`, \`execute_query\`, \`get_status\`, \`compile_model\`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
327068
+
327069
+ Most packages publish a curated surface, so a source being absent or refused is usually the package working as its author intended, not a fault. Read the refusal before working around it, because the three things that can refuse you need three different responses and they are told apart by the status code.
327070
+
327071
+ ## The three answers
327072
+
327073
+ | what you got | what it means | what to do |
327074
+ | --- | --- | --- |
327075
+ | **404**, "No queryable source", "No queryable model" or "Query target is not queryable" | the source is not on the package's published surface, or does not exist. The two are deliberately indistinguishable so that a refusal cannot be used to probe for hidden names | query something on the surface, or ask the package's author to publish it |
327076
+ | **403**, "Access denied" | the source exists and is on the surface, but an \`#(authorize)\` gate did not admit you | supply the givens the gate reads, or accept that this caller may not read it |
327077
+ | **200** with zero rows | you were admitted, and an \`#(access_filter)\` narrowed the rows to none of them | this is a real answer. Report it as "no matching rows", never as an error |
327078
+
327079
+ A 403 names a source, so it tells you the source exists. A 404 tells you nothing at all. That asymmetry is deliberate: a source that is both hidden AND gated answers **404**, so you can never use a 403 to discover that a hidden name is real.
327080
+
327081
+ ## Addressing the surface, which is the most common 404
327082
+
327083
+ A query names a model file. When a package curates its surface, only files on that surface are valid entry points, **even for a source that file declares itself**:
327084
+
327085
+ \`\`\`
327086
+ POST .../packages/sales/models/index.malloy/query {"sourceName": "orders"} -> 200
327087
+ POST .../packages/sales/models/orders.malloy/query {"sourceName": "orders"} -> 404
327088
+ \`\`\`
327089
+
327090
+ Same source, same package. \`orders.malloy\` is off the surface, so it is not an entry point.
327091
+
327092
+ **Use the \`model_path\` that \`get_context\` gave you, verbatim.** Its \`source_info.resource_id\` holds \`environment\`, \`package\`, \`model_path\` and \`source\`, and those are exactly \`execute_query\`'s \`environmentName\`, \`packageName\`, \`modelPath\` and \`sourceName\`. On a curated package the \`model_path\` is the surface file, not the file that declares the source. Substituting the declaring file because it looks more correct is how this 404 happens.
327093
+
327094
+ ## When the source is not in \`get_context\` at all
327095
+
327096
+ Work down this list. The first three are far more common than the last.
327097
+
327098
+ 1. **It is off the surface.** The package publishes a curated set and this source is not in it. Nothing you can do from the query side; it is a package edit.
327099
+ 2. **The package failed to load, or is serving a stale model.** Call \`get_status\`. A package that never loaded is absent from listings entirely, which reads exactly like "does not exist". A package whose last reload failed to compile is listed and answering, from the model it compiled *before* that save, and carries \`stale: true\`. Neither is visible from a listing alone.
327100
+ 3. **You are talking to a different server than you think.** Call \`list_packages\` and check the environment and package names are the ones you expect. A stale \`.mcp.json\` outlives the server that wrote it, and another Publisher may hold that port.
327101
+ 4. **Your search phrasing missed it.** Retry with a bare target (no \`search_text\`) to enumerate rather than rank. If it appears there, the source exists and was a ranking miss.
327102
+
327103
+ ## What curation does not do
327104
+
327105
+ - **It does not hide fields.** A published source may \`join\` an unpublished one, and a query grouping by a joined field returns that field's values normally. Hiding a source removes it as a landing point; it does not redact columns a published source pulls in.
327106
+ - **It does not gate \`/compile\`.** \`compile_model\` is exempt, because compile is the authoring loop. A hidden source can still be compile-checked, and that is intended. The exception is a hidden source that is also gated, which answers 404 at compile too.
327107
+ - **It is not access control.** Curation answers "what is queryable by name", not "who may query it". Identity is \`#(authorize)\` and \`#(access_filter)\`. A source is not protected by being hidden.
327108
+
327109
+ ## Reaching an unpublished source from your own model
327110
+
327111
+ If you are writing Malloy rather than just querying, an unpublished source is still usable, with one condition that trips people.
327112
+
327113
+ \`export { ... }\` decides what an **importing file** can see, which is Malloy's rule and not Publisher's:
327114
+
327115
+ - a file that declares **no** \`export\` hands an importer everything it declares, so import the file that declares the source and join it normally;
327116
+ - a file that **does** declare one hands over exactly that list. Importing a file whose \`export\` omits the source and then naming it fails to compile with \`Reference to undefined object\`.
327117
+
327118
+ So an unpublished source is reachable through a file that exports it, or that exports nothing, and not through a file whose \`export\` leaves it out. Re-export chains are fine at any depth: a source re-exported through several files stays queryable through the surface, because admission follows the declaration rather than the path taken to it.
327119
+
327120
+ ## Before you report a source as missing
327121
+
327122
+ Confirm all three, in this order. Each one has been mistaken for "the data is not there".
327123
+
327124
+ 1. \`get_status\` shows the package loaded and not \`stale\`.
327125
+ 2. \`list_packages\` names the environment and package you meant.
327126
+ 3. An enumerating \`get_context\` call (a target with no \`search_text\`) does not list it.
327127
+
327128
+ Only then is it genuinely not published, and the answer to the user is that the package does not expose it, not that the data does not exist.`
326307
327129
  }
326308
327130
  ]
326309
327131
  };
@@ -327699,7 +328521,7 @@ async function withBigQueryLookupRetry(work, parentJobId) {
327699
328521
  parentJobId,
327700
328522
  error: error instanceof Error ? error.message : String(error)
327701
328523
  });
327702
- await new Promise((resolve6) => setTimeout(resolve6, BIGQUERY_LOOKUP_BACKOFF_MS * attempt));
328524
+ await new Promise((resolve7) => setTimeout(resolve7, BIGQUERY_LOOKUP_BACKOFF_MS * attempt));
327703
328525
  }
327704
328526
  }
327705
328527
  }
@@ -330384,11 +331206,11 @@ var promisifyStore = (passedStore) => {
330384
331206
 
330385
331207
  class PromisifiedStore {
330386
331208
  async increment(key) {
330387
- return new Promise((resolve6, reject2) => {
331209
+ return new Promise((resolve7, reject2) => {
330388
331210
  legacyStore.incr(key, (error, totalHits, resetTime) => {
330389
331211
  if (error)
330390
331212
  reject2(error);
330391
- resolve6({ totalHits, resetTime });
331213
+ resolve7({ totalHits, resetTime });
330392
331214
  });
330393
331215
  });
330394
331216
  }
@@ -330518,9 +331340,9 @@ var rateLimit = (passedOptions) => {
330518
331340
  }
330519
331341
  }
330520
331342
  const middleware = handleAsyncErrors(async (request, response, next) => {
330521
- const closePromise = config.skipFailedRequests && new Promise((resolve6) => response.once("close", resolve6));
330522
- const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((resolve6) => response.once("finish", resolve6));
330523
- const errorPromise = config.skipFailedRequests && new Promise((resolve6) => response.once("error", resolve6));
331343
+ const closePromise = config.skipFailedRequests && new Promise((resolve7) => response.once("close", resolve7));
331344
+ const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((resolve7) => response.once("finish", resolve7));
331345
+ const errorPromise = config.skipFailedRequests && new Promise((resolve7) => response.once("error", resolve7));
330524
331346
  debug2("requested %o", request.originalUrl);
330525
331347
  debug2("request from ip %o", request.ip);
330526
331348
  const skip = await config.skip(request, response);
@@ -331862,8 +332684,8 @@ mainServer.timeout = 600000;
331862
332684
  mainServer.keepAliveTimeout = 600000;
331863
332685
  mainServer.headersTimeout = 600000;
331864
332686
  var resolveRestBound = () => {};
331865
- var restBound = new Promise((resolve7) => {
331866
- resolveRestBound = resolve7;
332687
+ var restBound = new Promise((resolve8) => {
332688
+ resolveRestBound = resolve8;
331867
332689
  });
331868
332690
  mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
331869
332691
  resolveRestBound();