@malloy-publisher/server 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +125 -59
- package/dist/app/assets/{EnvironmentPage-B02geyxh.js → EnvironmentPage-ufeWfqnK.js} +1 -1
- package/dist/app/assets/{HomePage-BdKzZEto.js → HomePage-CN_FgmtD.js} +1 -1
- package/dist/app/assets/{LightMode-BRAs1k98.js → LightMode-BFbC4W45.js} +1 -1
- package/dist/app/assets/{MainPage-D9XZbGvk.js → MainPage-BcCyzakx.js} +1 -1
- package/dist/app/assets/{ModelPage-B-U47DzF.js → ModelPage-wHHnxoih.js} +2 -2
- package/dist/app/assets/{PackagePage-CDp9A3_k.js → PackagePage-DwdO-2dp.js} +1 -1
- package/dist/app/assets/{RenderedResult-CTD2Ej2S.es-BZsZAiwo.js → RenderedResult-CYyhQekT.es-DBHF8EwO.js} +2 -2
- package/dist/app/assets/{RouteError-CSsLG99c.js → RouteError-Blt6a2Za.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-C4u5l49i.js → ThemeEditorPage-Cb-2YbfG.js} +1 -1
- package/dist/app/assets/{ToggleButtonGroup-IBhUYkv_.js → ToggleButtonGroup-BuhURn8v.js} +1 -1
- package/dist/app/assets/{index-DOXSuS-a.js → index-BELjGKhI.js} +1 -1
- package/dist/app/assets/{index--DAcpR5Y.js → index-BJyLY8XQ.js} +1 -1
- package/dist/app/assets/{index-BKNRDF5y.js → index-DCizS2BO.js} +1 -1
- package/dist/app/assets/{index-Bd7WYNw_.js → index-Dz4y_2PE.js} +4 -4
- package/dist/app/assets/{index-CshLRozl.js → index-kt4oTEGz.js} +1 -1
- package/dist/app/assets/{index.es-Bj77sE9l.js → index.es-u5PRRTzT.js} +4 -4
- package/dist/app/index.html +1 -1
- package/dist/cpufeatures-1yrn0vtw.node +0 -0
- package/dist/package_load_worker.mjs +747 -357
- package/dist/server.mjs +1450 -628
- package/dist/{sshcrypto-xqan60jb.node → sshcrypto-cqb52edz.node} +0 -0
- package/package.json +1 -1
package/dist/server.mjs
CHANGED
|
@@ -175849,7 +175849,7 @@ import os from "os";
|
|
|
175849
175849
|
function normalizeModelPath(p) {
|
|
175850
175850
|
return p.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
175851
175851
|
}
|
|
175852
|
-
var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
|
|
175852
|
+
var API_PREFIX = "/api/v0", README_NAME = "README.md", PUBLISHER_CONFIG_NAME = "publisher.config.json", PACKAGE_MANIFEST_NAME = "publisher.json", MODEL_FILE_SUFFIX = ".malloy", NOTEBOOK_FILE_SUFFIX = ".malloynb", INDEX_MODEL_NAME = "index.malloy", DEFAULT_QUERY_ROW_LIMIT = 1000, DEFAULT_MAX_QUERY_ROWS = 1e5, DEFAULT_MAX_RESPONSE_BYTES = 50000000, DEFAULT_QUERY_TIMEOUT_MS = 300000, DEFAULT_MAX_CONCURRENT_QUERIES = 32, TEMP_DIR_PATH, PUBLISHER_DATA_DIR = "publisher_data";
|
|
175853
175853
|
var init_constants = __esm(() => {
|
|
175854
175854
|
TEMP_DIR_PATH = os.tmpdir();
|
|
175855
175855
|
});
|
|
@@ -232535,12 +232535,23 @@ var require_bcrypt_pbkdf = __commonJS((exports, module) => {
|
|
|
232535
232535
|
};
|
|
232536
232536
|
});
|
|
232537
232537
|
|
|
232538
|
+
// ../../node_modules/cpu-features/build/Release/cpufeatures.node
|
|
232539
|
+
var require_cpufeatures = __commonJS((exports, module) => {
|
|
232540
|
+
module.exports = __require("./cpufeatures-1yrn0vtw.node");
|
|
232541
|
+
});
|
|
232542
|
+
|
|
232543
|
+
// ../../node_modules/cpu-features/lib/index.js
|
|
232544
|
+
var require_lib8 = __commonJS((exports, module) => {
|
|
232545
|
+
var binding = require_cpufeatures();
|
|
232546
|
+
module.exports = binding.getCPUInfo;
|
|
232547
|
+
});
|
|
232548
|
+
|
|
232538
232549
|
// ../../node_modules/ssh2/lib/protocol/constants.js
|
|
232539
232550
|
var require_constants42 = __commonJS((exports, module) => {
|
|
232540
232551
|
var crypto3 = __require("crypto");
|
|
232541
232552
|
var cpuInfo;
|
|
232542
232553
|
try {
|
|
232543
|
-
cpuInfo = (
|
|
232554
|
+
cpuInfo = require_lib8()();
|
|
232544
232555
|
} catch {}
|
|
232545
232556
|
var { bindingAvailable, CIPHER_INFO, MAC_INFO } = require_crypto();
|
|
232546
232557
|
var eddsaSupported = (() => {
|
|
@@ -233139,7 +233150,7 @@ var require_utils92 = __commonJS((exports, module) => {
|
|
|
233139
233150
|
|
|
233140
233151
|
// ../../node_modules/ssh2/lib/protocol/crypto/build/Release/sshcrypto.node
|
|
233141
233152
|
var require_sshcrypto = __commonJS((exports, module) => {
|
|
233142
|
-
module.exports = __require("./sshcrypto-
|
|
233153
|
+
module.exports = __require("./sshcrypto-cqb52edz.node");
|
|
233143
233154
|
});
|
|
233144
233155
|
|
|
233145
233156
|
// ../../node_modules/ssh2/lib/protocol/crypto/poly1305.js
|
|
@@ -247183,7 +247194,7 @@ var require_keygen = __commonJS((exports, module) => {
|
|
|
247183
247194
|
});
|
|
247184
247195
|
|
|
247185
247196
|
// ../../node_modules/ssh2/lib/index.js
|
|
247186
|
-
var
|
|
247197
|
+
var require_lib9 = __commonJS((exports, module) => {
|
|
247187
247198
|
var {
|
|
247188
247199
|
AgentProtocol,
|
|
247189
247200
|
BaseAgent,
|
|
@@ -247366,7 +247377,7 @@ function openSshProxy(ssh, target) {
|
|
|
247366
247377
|
var import_ssh2, ALLOW_UNVERIFIED_SSH_HOST_KEY_ENV = "PUBLISHER_ALLOW_UNVERIFIED_SSH_HOST_KEY", TRUTHY_ENV_VALUES, SSH_CONNECT_TIMEOUT_MS = 15000, SSH_KEEPALIVE_INTERVAL_MS = 15000;
|
|
247367
247378
|
var init_proxy = __esm(() => {
|
|
247368
247379
|
init_logger();
|
|
247369
|
-
import_ssh2 = __toESM(
|
|
247380
|
+
import_ssh2 = __toESM(require_lib9(), 1);
|
|
247370
247381
|
TRUTHY_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
247371
247382
|
});
|
|
247372
247383
|
|
|
@@ -267096,6 +267107,29 @@ var require_lunr = __commonJS((exports, module) => {
|
|
|
267096
267107
|
})();
|
|
267097
267108
|
});
|
|
267098
267109
|
|
|
267110
|
+
// src/service/authorize_routes.ts
|
|
267111
|
+
function canonicalAuthorizeRoute(route) {
|
|
267112
|
+
switch (route) {
|
|
267113
|
+
case ACCESS_FILTER_ROUTE:
|
|
267114
|
+
return ACCESS_FILTER_ROUTE;
|
|
267115
|
+
case AUTHORIZE_ROUTE:
|
|
267116
|
+
return AUTHORIZE_ROUTE;
|
|
267117
|
+
default:
|
|
267118
|
+
return;
|
|
267119
|
+
}
|
|
267120
|
+
}
|
|
267121
|
+
var AUTHORIZE_ROUTE = "authorize", ACCESS_FILTER_ROUTE = "access_filter", RECOGNIZED_AUTHORIZE_SPELLINGS, CANONICAL_AUTHORIZE_ROUTES;
|
|
267122
|
+
var init_authorize_routes = __esm(() => {
|
|
267123
|
+
RECOGNIZED_AUTHORIZE_SPELLINGS = [
|
|
267124
|
+
ACCESS_FILTER_ROUTE,
|
|
267125
|
+
AUTHORIZE_ROUTE
|
|
267126
|
+
];
|
|
267127
|
+
CANONICAL_AUTHORIZE_ROUTES = [
|
|
267128
|
+
ACCESS_FILTER_ROUTE,
|
|
267129
|
+
AUTHORIZE_ROUTE
|
|
267130
|
+
];
|
|
267131
|
+
});
|
|
267132
|
+
|
|
267099
267133
|
// src/service/authorize.ts
|
|
267100
267134
|
import { payloadOf, routeOf } from "@malloydata/malloy";
|
|
267101
267135
|
function noteRoute(text) {
|
|
@@ -267104,8 +267138,61 @@ function noteRoute(text) {
|
|
|
267104
267138
|
function notePayload(text) {
|
|
267105
267139
|
return payloadOf({ value: text.trimStart() }) ?? "";
|
|
267106
267140
|
}
|
|
267141
|
+
function nearMissRouteNames(route) {
|
|
267142
|
+
switch (route) {
|
|
267143
|
+
case AUTHORIZE_ROUTE:
|
|
267144
|
+
return [
|
|
267145
|
+
"authorize",
|
|
267146
|
+
"source_authorize",
|
|
267147
|
+
"source-authorize",
|
|
267148
|
+
"sourceauthorize",
|
|
267149
|
+
"authorize_source",
|
|
267150
|
+
"authorize-source",
|
|
267151
|
+
"authorizesource"
|
|
267152
|
+
];
|
|
267153
|
+
case ACCESS_FILTER_ROUTE:
|
|
267154
|
+
return [
|
|
267155
|
+
"access_filter",
|
|
267156
|
+
"access-filter",
|
|
267157
|
+
"accessfilter",
|
|
267158
|
+
"filter_access",
|
|
267159
|
+
"filter-access",
|
|
267160
|
+
"filteraccess",
|
|
267161
|
+
"row_authorize",
|
|
267162
|
+
"row-authorize",
|
|
267163
|
+
"rowauthorize",
|
|
267164
|
+
"authorize_row",
|
|
267165
|
+
"authorize-row",
|
|
267166
|
+
"authorizerow"
|
|
267167
|
+
];
|
|
267168
|
+
default:
|
|
267169
|
+
return [route];
|
|
267170
|
+
}
|
|
267171
|
+
}
|
|
267172
|
+
function nearMissWordAlternation(route) {
|
|
267173
|
+
const names = nearMissRouteNames(route);
|
|
267174
|
+
if (names.length === 0) {
|
|
267175
|
+
throw new Error(`No near-miss spellings for authorize route \`${route}\` — an empty ` + "alternation matches every malformed-prefix note, which would " + "fail package loads over annotations nobody wrote.");
|
|
267176
|
+
}
|
|
267177
|
+
return names.map((name) => name.replace(REGEXP_SPECIAL, "\\$&")).join("|");
|
|
267178
|
+
}
|
|
267179
|
+
function motlyAuthorizePayloadPattern(route) {
|
|
267180
|
+
return new RegExp(`^[ \\t]*[([{<][ \\t]*(?:${nearMissWordAlternation(route)})[ \\t]*[)\\]}>]`, "iu");
|
|
267181
|
+
}
|
|
267182
|
+
function malformedAuthorizeAttemptPattern(route) {
|
|
267183
|
+
return new RegExp(`^##?\\|?[ \\t]*[([{<]?[ \\t]*(?:${nearMissWordAlternation(route)})`, "iu");
|
|
267184
|
+
}
|
|
267107
267185
|
function authorizeNoteContent(text) {
|
|
267108
|
-
|
|
267186
|
+
const route = canonicalAuthorizeRoute(noteRoute(text));
|
|
267187
|
+
if (route === undefined)
|
|
267188
|
+
return;
|
|
267189
|
+
return { route, content: notePayload(text) };
|
|
267190
|
+
}
|
|
267191
|
+
function authorizeAnnotationRoute(text) {
|
|
267192
|
+
return authorizeNoteContent(text)?.route;
|
|
267193
|
+
}
|
|
267194
|
+
function hasCallerAuthorizeAnnotation(callerText) {
|
|
267195
|
+
return AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText);
|
|
267109
267196
|
}
|
|
267110
267197
|
function assertNoCallerAuthorizeAnnotation(callerText) {
|
|
267111
267198
|
if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
|
|
@@ -267115,37 +267202,59 @@ function assertNoCallerAuthorizeAnnotation(callerText) {
|
|
|
267115
267202
|
function containsAuthorizeAnnotationTag(texts) {
|
|
267116
267203
|
return texts.some((text) => authorizeNoteContent(text) !== undefined);
|
|
267117
267204
|
}
|
|
267118
|
-
function collectAuthorizeNearMisses(texts) {
|
|
267205
|
+
function collectAuthorizeNearMisses(texts, route) {
|
|
267206
|
+
const motlyPattern = motlyAuthorizePayloadPattern(route);
|
|
267207
|
+
const malformedPattern = malformedAuthorizeAttemptPattern(route);
|
|
267208
|
+
const nearMissNames = new Set(nearMissRouteNames(route).map((name) => name.toLowerCase()));
|
|
267119
267209
|
const found = [];
|
|
267120
267210
|
for (const text of texts) {
|
|
267121
267211
|
const trimmed = text.trimStart();
|
|
267122
|
-
const
|
|
267123
|
-
if (
|
|
267212
|
+
const noted = noteRoute(trimmed);
|
|
267213
|
+
if (noted === route)
|
|
267124
267214
|
continue;
|
|
267125
|
-
const nearMiss =
|
|
267215
|
+
const nearMiss = noted === undefined ? malformedPattern.test(trimmed) : noted === "" ? motlyPattern.test(notePayload(trimmed)) : nearMissNames.has(noted.toLowerCase());
|
|
267126
267216
|
if (!nearMiss)
|
|
267127
267217
|
continue;
|
|
267128
267218
|
found.push(trimmed.split(/[\r\n]/, 1)[0]);
|
|
267129
267219
|
}
|
|
267130
267220
|
return found;
|
|
267131
267221
|
}
|
|
267222
|
+
function collectAuthorizeNearMissesAllRoutes(texts) {
|
|
267223
|
+
const materialized = [...texts];
|
|
267224
|
+
return RECOGNIZED_AUTHORIZE_SPELLINGS.flatMap((route) => collectAuthorizeNearMisses(materialized, route));
|
|
267225
|
+
}
|
|
267226
|
+
function guessedNearMissRoute(text) {
|
|
267227
|
+
const lower = text.toLowerCase();
|
|
267228
|
+
let best = ACCESS_FILTER_ROUTE;
|
|
267229
|
+
let bestLength = 0;
|
|
267230
|
+
for (const route of RECOGNIZED_AUTHORIZE_SPELLINGS) {
|
|
267231
|
+
for (const name of nearMissRouteNames(route)) {
|
|
267232
|
+
if (lower.includes(name) && name.length > bestLength) {
|
|
267233
|
+
best = route;
|
|
267234
|
+
bestLength = name.length;
|
|
267235
|
+
}
|
|
267236
|
+
}
|
|
267237
|
+
}
|
|
267238
|
+
return best;
|
|
267239
|
+
}
|
|
267132
267240
|
function assertNoAuthorizeNearMisses(found) {
|
|
267133
267241
|
if (found.length === 0)
|
|
267134
267242
|
return;
|
|
267135
267243
|
const unique = [...new Set(found)];
|
|
267136
267244
|
throw new ModelCompilationError({
|
|
267137
|
-
message: `These annotations are
|
|
267138
|
-
${unique.map((t) => ` - \`${t}
|
|
267245
|
+
message: `These annotations are never enforced:
|
|
267246
|
+
${unique.map((t) => ` - \`${t}\` (meant \`#(${guessedNearMissRoute(t)})\`?)`).join(`
|
|
267139
267247
|
`)}
|
|
267140
|
-
` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\`
|
|
267248
|
+
` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\` and \`#(access_filter)\` reach an access-control ` + `route (in their \`##\` and ` + `block-form \`#|(...)\` spellings too). A space after the \`#\`, spaces ` + `inside the brackets, a mis-hyphenated or transposed word, or anything ` + `trailing the closing bracket makes it a plain tag Malloy hands to ` + `something else. Write the tag named above, unquoted, on its own line ` + `directly above the \`source:\` statement you mean to protect (the ` + `quoted string form is retired). This is refused rather than ` + `interpreted: guessing at the intent would let publisher start ` + `enforcing a filter on a package that has been serving every row.`
|
|
267141
267249
|
});
|
|
267142
267250
|
}
|
|
267143
267251
|
function describeMisplacedAuthorizeAnnotation(f) {
|
|
267144
|
-
if (f.kind === "query")
|
|
267145
|
-
return `on query "${f.name}"`;
|
|
267146
267252
|
if (f.kind === "file")
|
|
267147
|
-
return
|
|
267148
|
-
|
|
267253
|
+
return `at the file level (\`##(${f.route})\`)`;
|
|
267254
|
+
const tag = `\`#(${f.route})\``;
|
|
267255
|
+
if (f.kind === "query")
|
|
267256
|
+
return `on query "${f.name}" (${tag})`;
|
|
267257
|
+
return `on field "${f.fieldName}" of source "${f.name}" (${tag})`;
|
|
267149
267258
|
}
|
|
267150
267259
|
function assertNoMisplacedAuthorizeAnnotations(found) {
|
|
267151
267260
|
if (found.length === 0)
|
|
@@ -267153,9 +267262,9 @@ function assertNoMisplacedAuthorizeAnnotations(found) {
|
|
|
267153
267262
|
const positions = found.map((f) => ` - ${describeMisplacedAuthorizeAnnotation(f)}`).join(`
|
|
267154
267263
|
`);
|
|
267155
267264
|
throw new ModelCompilationError({
|
|
267156
|
-
message: `An
|
|
267265
|
+
message: `An authorize annotation is never enforced at:
|
|
267157
267266
|
${positions}
|
|
267158
|
-
` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base.
|
|
267267
|
+
` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base. A file-level annotation (\`##(authorize)\` / ` + `\`##(access_filter)\`) is deprecated and no longer enforced ` + `anywhere, so it always lands here: declare the same tag on each ` + `\`source:\` it was meant to protect instead. Every other ` + `position above should move — with the same tag named beside it — ` + `to the \`source:\` statement it is meant ` + `to protect.`
|
|
267159
267268
|
});
|
|
267160
267269
|
}
|
|
267161
267270
|
function referencedGivenNames(expr) {
|
|
@@ -267192,7 +267301,7 @@ function liftProbeFilterCondition(prepared, label, filterText) {
|
|
|
267192
267301
|
return lifted;
|
|
267193
267302
|
}
|
|
267194
267303
|
function gateFilterText(exprs) {
|
|
267195
|
-
return exprs.map((e) => `(${e})`).join("
|
|
267304
|
+
return exprs.map((e) => `(${e})`).join(" and ");
|
|
267196
267305
|
}
|
|
267197
267306
|
async function liftRowLevelCondition(compiler, sourceName, exprs) {
|
|
267198
267307
|
const filterText = gateFilterText(exprs);
|
|
@@ -267202,7 +267311,7 @@ async function liftRowLevelCondition(compiler, sourceName, exprs) {
|
|
|
267202
267311
|
async function validateAuthorizeProbes(compiler, options) {
|
|
267203
267312
|
const ownNotesOf = options.authorizeOwnNotes ?? new Map;
|
|
267204
267313
|
for (const [sourceName, groups] of options.authorizeMap ?? []) {
|
|
267205
|
-
for (const exprs of groups) {
|
|
267314
|
+
for (const { route, exprs } of groups) {
|
|
267206
267315
|
if (exprs.length === 0)
|
|
267207
267316
|
continue;
|
|
267208
267317
|
let condition;
|
|
@@ -267210,17 +267319,17 @@ async function validateAuthorizeProbes(compiler, options) {
|
|
|
267210
267319
|
condition = await liftRowLevelCondition(compiler, sourceName, exprs);
|
|
267211
267320
|
} catch (err) {
|
|
267212
267321
|
const detail = err instanceof Error ? err.message : String(err);
|
|
267213
|
-
const ownNotes = ownNotesOf.get(sourceName) ?? [];
|
|
267322
|
+
const ownNotes = ownNotesOf.get(sourceName)?.get(route) ?? [];
|
|
267214
267323
|
if (ownNotes.length === 0) {
|
|
267215
267324
|
options.onRowLevelGateRejected?.("entry_point_unexpressible");
|
|
267216
267325
|
options.onRowLevelGateUnexpressible?.(sourceName, detail);
|
|
267217
267326
|
continue;
|
|
267218
267327
|
}
|
|
267219
267328
|
throw new ModelCompilationError({
|
|
267220
|
-
message: `Invalid #(
|
|
267329
|
+
message: `Invalid #(${route}) annotation on source "${sourceName}" ` + `[${exprs.join(" and ")}]: ${detail}`
|
|
267221
267330
|
});
|
|
267222
267331
|
}
|
|
267223
|
-
options.onOwnRowLevelConditionCompiled?.(sourceName, condition);
|
|
267332
|
+
options.onOwnRowLevelConditionCompiled?.(sourceName, condition, route);
|
|
267224
267333
|
}
|
|
267225
267334
|
}
|
|
267226
267335
|
}
|
|
@@ -267234,59 +267343,41 @@ function unquoteLegacyGatePayload(payload) {
|
|
|
267234
267343
|
return trimmed[0] === "'" ? inner.replace(SINGLE_QUOTE_ESCAPE, "$1") : inner.replace(DOUBLE_QUOTE_ESCAPE, "$1");
|
|
267235
267344
|
}
|
|
267236
267345
|
function parseAuthorizeAnnotation(annotation) {
|
|
267237
|
-
const
|
|
267238
|
-
if (
|
|
267346
|
+
const parsed = authorizeNoteContent(annotation);
|
|
267347
|
+
if (parsed === undefined)
|
|
267239
267348
|
return null;
|
|
267240
|
-
const trimmed = content.trim();
|
|
267349
|
+
const trimmed = parsed.content.trim();
|
|
267241
267350
|
if (trimmed.length === 0) {
|
|
267242
267351
|
throw new Error("authorize annotation has an empty expression body");
|
|
267243
267352
|
}
|
|
267244
|
-
return trimmed;
|
|
267353
|
+
return { route: parsed.route, expr: trimmed };
|
|
267245
267354
|
}
|
|
267246
267355
|
function collectAuthorizeExprs(annotations) {
|
|
267247
267356
|
const exprs = [];
|
|
267248
267357
|
for (const annotation of annotations) {
|
|
267249
|
-
const
|
|
267250
|
-
if (
|
|
267251
|
-
exprs.push(
|
|
267358
|
+
const parsed = parseAuthorizeAnnotation(annotation);
|
|
267359
|
+
if (parsed !== null) {
|
|
267360
|
+
exprs.push(parsed);
|
|
267252
267361
|
}
|
|
267253
267362
|
}
|
|
267254
267363
|
return exprs;
|
|
267255
267364
|
}
|
|
267256
|
-
function
|
|
267257
|
-
|
|
267258
|
-
for (const [sourceName, notes] of authorizeOwnNotes) {
|
|
267259
|
-
if (notes.length > 1) {
|
|
267260
|
-
found.push({ sourceName, texts: notes.map((note) => note.text) });
|
|
267261
|
-
}
|
|
267262
|
-
}
|
|
267263
|
-
return found;
|
|
267264
|
-
}
|
|
267265
|
-
function assertAtMostOneAuthorizeGate(found) {
|
|
267266
|
-
if (found.length === 0)
|
|
267267
|
-
return;
|
|
267268
|
-
const positions = found.map(({ sourceName, texts }) => ` - source "${sourceName}" declares ${texts.length}:
|
|
267269
|
-
` + texts.map((t) => ` \`${t.trim().split(/[\r\n]/, 1)[0]}\``).join(`
|
|
267270
|
-
`)).join(`
|
|
267271
|
-
`);
|
|
267272
|
-
throw new ModelCompilationError({
|
|
267273
|
-
message: `A source may declare at most one \`#(authorize)\` annotation:
|
|
267274
|
-
${positions}
|
|
267275
|
-
` + `Combine multiple conditions into one expression with \`or\` instead of ` + `repeating the annotation, e.g. ` + "`#(authorize) $ROLE = 'admin' or org_id in $GROUPS`."
|
|
267276
|
-
});
|
|
267365
|
+
function collectAuthorizeExprsForRoute(annotations, route) {
|
|
267366
|
+
return collectAuthorizeExprs(annotations).filter((e) => e.route === route).map((e) => e.expr);
|
|
267277
267367
|
}
|
|
267278
267368
|
function findLegacyStringGates(authorizeOwnNotes) {
|
|
267279
267369
|
const found = [];
|
|
267280
|
-
for (const [sourceName,
|
|
267370
|
+
for (const [sourceName, notesByRoute] of authorizeOwnNotes) {
|
|
267371
|
+
const notes = [...notesByRoute.values()].flat();
|
|
267281
267372
|
const legacyNotes = notes.filter((note) => {
|
|
267282
267373
|
const content = authorizeNoteContent(note.text);
|
|
267283
|
-
return content !== undefined && isLegacyQuotedPayload(content);
|
|
267374
|
+
return content !== undefined && isLegacyQuotedPayload(content.content);
|
|
267284
267375
|
});
|
|
267285
267376
|
if (legacyNotes.length === 0)
|
|
267286
267377
|
continue;
|
|
267287
|
-
const
|
|
267288
|
-
if (
|
|
267289
|
-
found.push({ sourceName,
|
|
267378
|
+
const gates = legacyNotes.flatMap((note) => collectAuthorizeExprs([note.text]));
|
|
267379
|
+
if (gates.length > 0) {
|
|
267380
|
+
found.push({ sourceName, gates });
|
|
267290
267381
|
}
|
|
267291
267382
|
}
|
|
267292
267383
|
return found;
|
|
@@ -267294,22 +267385,22 @@ function findLegacyStringGates(authorizeOwnNotes) {
|
|
|
267294
267385
|
function assertNoLegacyStringGate(found) {
|
|
267295
267386
|
if (found.length === 0)
|
|
267296
267387
|
return;
|
|
267297
|
-
const rewrites = found.flatMap(({ sourceName,
|
|
267298
|
-
#(
|
|
267388
|
+
const rewrites = found.flatMap(({ sourceName, gates }) => gates.map(({ route, expr }) => ` - source "${sourceName}": replace \`#(${route}) ${expr}\`` + ` with
|
|
267389
|
+
#(${route}) ${unquoteLegacyGatePayload(expr)}`)).join(`
|
|
267299
267390
|
`);
|
|
267300
267391
|
throw new ModelCompilationError({
|
|
267301
|
-
message: `The string form of
|
|
267392
|
+
message: `The string form of an authorize gate (a Malloy-quoted expression on ` + `the \`source:\` line) is no longer accepted. Replace it with the ` + `unquoted expression, carried by the same annotation on its own line ` + `directly above the \`source:\` line:
|
|
267302
267393
|
${rewrites}
|
|
267303
267394
|
` + `Test that the rewrite gates the same rows and check the row count ` + `matches what the string form served.`
|
|
267304
267395
|
});
|
|
267305
267396
|
}
|
|
267306
|
-
var
|
|
267397
|
+
var AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, REGEXP_SPECIAL, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
|
|
267307
267398
|
var init_authorize = __esm(() => {
|
|
267308
267399
|
init_errors();
|
|
267309
|
-
|
|
267400
|
+
init_authorize_routes();
|
|
267401
|
+
AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*[([{<]?[ \t]*(?:(?:(?:row|source)[-_]?)?authorize|access[-_]?filter)(?=[)\]}>]|[ \t]|$)`;
|
|
267310
267402
|
AUTHORIZE_ANNOTATION_ANYWHERE = new RegExp(AUTHORIZE_TAG_LIKE, "iu");
|
|
267311
|
-
|
|
267312
|
-
MALFORMED_AUTHORIZE_ATTEMPT = /^##?\|?[ \t]*[([{<]?[ \t]*authorize/iu;
|
|
267403
|
+
REGEXP_SPECIAL = /[.*+?^${}()|[\]\\]/g;
|
|
267313
267404
|
GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
267314
267405
|
SINGLE_QUOTE_ESCAPE = /\\(['\\])/g;
|
|
267315
267406
|
DOUBLE_QUOTE_ESCAPE = /\\(["\\])/g;
|
|
@@ -267894,7 +267985,13 @@ function collectGivenRefs(value, into) {
|
|
|
267894
267985
|
for (const child of Object.values(node))
|
|
267895
267986
|
collectGivenRefs(child, into);
|
|
267896
267987
|
}
|
|
267897
|
-
function
|
|
267988
|
+
function gateGivenSource(sources, name) {
|
|
267989
|
+
const source = sources.find((candidate) => candidate.name === name);
|
|
267990
|
+
if (!source)
|
|
267991
|
+
return;
|
|
267992
|
+
return [...source.authorize ?? [], ...source.accessFilter ?? []];
|
|
267993
|
+
}
|
|
267994
|
+
function suggestGivenLookup(modelDef, gatesBySource, surfaced) {
|
|
267898
267995
|
const registry = modelDef.givens ?? {};
|
|
267899
267996
|
const bySource = new Map;
|
|
267900
267997
|
const byQuery = new Map;
|
|
@@ -267903,7 +268000,7 @@ function suggestGivenLookup(modelDef, authorizeBySource, surfaced) {
|
|
|
267903
268000
|
const name = obj.as || obj.name;
|
|
267904
268001
|
const refs = new Set;
|
|
267905
268002
|
collectGivenRefs(obj.filterList, refs);
|
|
267906
|
-
for (const expr of
|
|
268003
|
+
for (const expr of gatesBySource(name) ?? []) {
|
|
267907
268004
|
for (const given of referencedGivenNames(expr))
|
|
267908
268005
|
refs.add(given);
|
|
267909
268006
|
}
|
|
@@ -268002,7 +268099,7 @@ function positiveInteger(value) {
|
|
|
268002
268099
|
function normalizeTileExpression(tile) {
|
|
268003
268100
|
return tile.split("->").map((part) => part.trim().replace(/\s+/g, " ")).join(" -> ").replace(/\s+/g, " ");
|
|
268004
268101
|
}
|
|
268005
|
-
function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames,
|
|
268102
|
+
function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, gatesBySource) {
|
|
268006
268103
|
const registry = modelDef.givens ?? {};
|
|
268007
268104
|
const surfaced = new Set(surfacedGivenNames);
|
|
268008
268105
|
const givens = new Map;
|
|
@@ -268072,7 +268169,7 @@ function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, author
|
|
|
268072
268169
|
viewAnnotations,
|
|
268073
268170
|
sourceFields,
|
|
268074
268171
|
drills,
|
|
268075
|
-
suggestGivens: suggestGivenLookup(modelDef, (source) =>
|
|
268172
|
+
suggestGivens: suggestGivenLookup(modelDef, (source) => gatesBySource?.get(source), surfaced)
|
|
268076
268173
|
};
|
|
268077
268174
|
}
|
|
268078
268175
|
function readTileLayout(annotations) {
|
|
@@ -270706,20 +270803,33 @@ function recordAuthorizeBypass(entryPoint) {
|
|
|
270706
270803
|
}
|
|
270707
270804
|
function recordRowLevelGateDecision(decision) {
|
|
270708
270805
|
rowLevelDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_total", {
|
|
270709
|
-
description: "How a row-level `#(
|
|
270806
|
+
description: "How a row-level `#(access_filter)` gate resolved a request. Label: decision ('denied_by_gate'|'empty_after_filter'). 'denied_by_gate' is the fail-closed refusal when the gate could not be applied; 'empty_after_filter' is a successful response with zero rows after the filter matched none, which is NOT an error."
|
|
270710
270807
|
});
|
|
270711
270808
|
rowLevelDecisionCounter.add(1, { decision });
|
|
270712
270809
|
}
|
|
270713
270810
|
function recordRowLevelGateRejected(cause) {
|
|
270714
270811
|
rowLevelRejectionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_rejected_total", {
|
|
270715
|
-
description: "Row-level `#(
|
|
270812
|
+
description: "Row-level `#(access_filter)` gates that were refused, warned about, or could not be resolved. Label: cause (" + ROW_LEVEL_GATE_REJECTION_CAUSES.map((c) => `'${c}'`).join("|") + "). Only 'legacy_string_gate' fails the whole model load. 'entry_point_unexpressible', 'source_line_gate_no_given_reference' and 'source_line_gate_negated_membership' warn at load and leave the model servable. 'unreachable_given', 'given_usage_unresolvable' and 'unclassifiable_condition' are request-time only — that entry point denies every request. Alert on any nonzero value since the last publish; the request-time causes also make this a rate signal, so see the doc above before writing an alert."
|
|
270716
270813
|
});
|
|
270717
270814
|
rowLevelRejectionCounter.add(1, { cause });
|
|
270718
270815
|
}
|
|
270719
|
-
|
|
270816
|
+
function recordAuthorizeAdmitAllGate(route) {
|
|
270817
|
+
admitAllCounter ??= publisherMeter().createCounter("publisher_authorize_admit_all_total", {
|
|
270818
|
+
description: "Sources declaring their OWN unconditional admit-all gate (`#(authorize) true`), counted once each at package load. Label: route (" + CANONICAL_AUTHORIZE_ROUTES.map((r) => `'${r}'`).join("|") + "). Not an error — `true` is the only spelling for an extension that deliberately re-opens a gated base — but it is the one declaration that turns a gate off, so a jump since the last publish is worth a look."
|
|
270819
|
+
});
|
|
270820
|
+
admitAllCounter.add(1, { route });
|
|
270821
|
+
}
|
|
270822
|
+
function recordLockDecision(decision) {
|
|
270823
|
+
lockDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_lock_total", {
|
|
270824
|
+
description: "How a `#(authorize)` lock resolved a request. Label: decision ('admitted'|'denied_by_lock'|'denied_unresolvable'). Both denials are a 403; 'denied_by_lock' is the gate refusing a caller it does not admit (routine), 'denied_unresolvable' is the fail-closed refusal when the gate could not be decided at all."
|
|
270825
|
+
});
|
|
270826
|
+
lockDecisionCounter.add(1, { decision });
|
|
270827
|
+
}
|
|
270828
|
+
var guardRejectionCounter = null, bypassCounter = null, rowLevelDecisionCounter = null, rowLevelRejectionCounter = null, admitAllCounter = null, lockDecisionCounter = null;
|
|
270720
270829
|
var init_authorize_metrics = __esm(() => {
|
|
270721
270830
|
init_telemetry();
|
|
270722
270831
|
init_authorize();
|
|
270832
|
+
init_authorize_routes();
|
|
270723
270833
|
});
|
|
270724
270834
|
|
|
270725
270835
|
// src/materialization_metrics.ts
|
|
@@ -271918,6 +272028,317 @@ function filterFreshManifest(entries, now) {
|
|
|
271918
272028
|
return { manifest, nextStaleSince };
|
|
271919
272029
|
}
|
|
271920
272030
|
|
|
272031
|
+
// src/service/authorize_grammar.ts
|
|
272032
|
+
import { routeOf as routeOf2 } from "@malloydata/malloy";
|
|
272033
|
+
function noteRoute2(text) {
|
|
272034
|
+
return routeOf2({ value: text.trimStart() });
|
|
272035
|
+
}
|
|
272036
|
+
function splitFieldPathSegments(fieldPath) {
|
|
272037
|
+
const segments = [];
|
|
272038
|
+
let i = 0;
|
|
272039
|
+
while (i < fieldPath.length) {
|
|
272040
|
+
if (fieldPath[i] === "`") {
|
|
272041
|
+
const end = fieldPath.indexOf("`", i + 1);
|
|
272042
|
+
segments.push(fieldPath.slice(i + 1, end));
|
|
272043
|
+
i = end + 1;
|
|
272044
|
+
} else {
|
|
272045
|
+
const dot = fieldPath.indexOf(".", i);
|
|
272046
|
+
const end = dot === -1 ? fieldPath.length : dot;
|
|
272047
|
+
segments.push(fieldPath.slice(i, end));
|
|
272048
|
+
i = end;
|
|
272049
|
+
}
|
|
272050
|
+
if (fieldPath[i] === ".")
|
|
272051
|
+
i++;
|
|
272052
|
+
}
|
|
272053
|
+
return segments;
|
|
272054
|
+
}
|
|
272055
|
+
function rejectionMessage(sourceName, body, detail) {
|
|
272056
|
+
return `Source "${sourceName}" declares an authorize gate \`${body}\`: ${detail} ` + GRAMMAR_SUMMARY;
|
|
272057
|
+
}
|
|
272058
|
+
function reject(sourceName, body, cause, detail) {
|
|
272059
|
+
throw new AuthorizeGrammarError(cause, rejectionMessage(sourceName, body, detail));
|
|
272060
|
+
}
|
|
272061
|
+
function rejectCoherence(sourceName, cause, detail) {
|
|
272062
|
+
throw new AuthorizeGrammarError(cause, `Source "${sourceName}" declares authorize notes whose terms ` + `conflict: ${detail} ${GRAMMAR_SUMMARY}`);
|
|
272063
|
+
}
|
|
272064
|
+
function splitTerms(body) {
|
|
272065
|
+
const terms = [];
|
|
272066
|
+
const compoundTokens = [];
|
|
272067
|
+
let depth = 0;
|
|
272068
|
+
let quote;
|
|
272069
|
+
let start = 0;
|
|
272070
|
+
let i = 0;
|
|
272071
|
+
while (i < body.length) {
|
|
272072
|
+
const ch = body[i];
|
|
272073
|
+
if (quote) {
|
|
272074
|
+
if (ch === "\\" && quote !== "`") {
|
|
272075
|
+
i += 2;
|
|
272076
|
+
continue;
|
|
272077
|
+
}
|
|
272078
|
+
if (ch === quote)
|
|
272079
|
+
quote = undefined;
|
|
272080
|
+
i++;
|
|
272081
|
+
continue;
|
|
272082
|
+
}
|
|
272083
|
+
if (isQuote(ch)) {
|
|
272084
|
+
quote = ch;
|
|
272085
|
+
i++;
|
|
272086
|
+
continue;
|
|
272087
|
+
}
|
|
272088
|
+
if (ch === "(") {
|
|
272089
|
+
depth++;
|
|
272090
|
+
i++;
|
|
272091
|
+
continue;
|
|
272092
|
+
}
|
|
272093
|
+
if (ch === ")") {
|
|
272094
|
+
depth = Math.max(0, depth - 1);
|
|
272095
|
+
i++;
|
|
272096
|
+
continue;
|
|
272097
|
+
}
|
|
272098
|
+
if (depth === 0) {
|
|
272099
|
+
const wordMatch = /^(and|or|not)\b/i.exec(body.slice(i));
|
|
272100
|
+
if (wordMatch && (i === 0 || !/[A-Za-z0-9_]/.test(body[i - 1]))) {
|
|
272101
|
+
const word = wordMatch[1].toLowerCase();
|
|
272102
|
+
if (word === "and") {
|
|
272103
|
+
terms.push(body.slice(start, i));
|
|
272104
|
+
i += wordMatch[0].length;
|
|
272105
|
+
start = i;
|
|
272106
|
+
continue;
|
|
272107
|
+
}
|
|
272108
|
+
compoundTokens.push(word);
|
|
272109
|
+
}
|
|
272110
|
+
}
|
|
272111
|
+
i++;
|
|
272112
|
+
}
|
|
272113
|
+
terms.push(body.slice(start));
|
|
272114
|
+
return { terms, compoundTokens };
|
|
272115
|
+
}
|
|
272116
|
+
function isQuote(ch) {
|
|
272117
|
+
return ch === "'" || ch === '"' || ch === "`";
|
|
272118
|
+
}
|
|
272119
|
+
function findOperatorOutsideQuotes(s, re) {
|
|
272120
|
+
let quote;
|
|
272121
|
+
for (let i = 0;i < s.length; i++) {
|
|
272122
|
+
const ch = s[i];
|
|
272123
|
+
if (quote) {
|
|
272124
|
+
if (ch === "\\" && quote !== "`") {
|
|
272125
|
+
i++;
|
|
272126
|
+
continue;
|
|
272127
|
+
}
|
|
272128
|
+
if (ch === quote)
|
|
272129
|
+
quote = undefined;
|
|
272130
|
+
continue;
|
|
272131
|
+
}
|
|
272132
|
+
if (isQuote(ch)) {
|
|
272133
|
+
quote = ch;
|
|
272134
|
+
continue;
|
|
272135
|
+
}
|
|
272136
|
+
re.lastIndex = i;
|
|
272137
|
+
const m = re.exec(s);
|
|
272138
|
+
if (m && m.index === i)
|
|
272139
|
+
return i;
|
|
272140
|
+
}
|
|
272141
|
+
return -1;
|
|
272142
|
+
}
|
|
272143
|
+
function parseTerm(sourceName, body, term, givenDeclaredTypes) {
|
|
272144
|
+
const trimmed2 = term.trim();
|
|
272145
|
+
if (findOperatorOutsideQuotes(trimmed2, NEGATED_OPERATOR_RE) !== -1) {
|
|
272146
|
+
reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
|
|
272147
|
+
}
|
|
272148
|
+
if (findOperatorOutsideQuotes(trimmed2, COMPARISON_OPERATOR_RE) !== -1) {
|
|
272149
|
+
reject(sourceName, body, "comparison_operator", "only `=` and `in` are allowed, not `<`/`>`/`<=`/`>=`.");
|
|
272150
|
+
}
|
|
272151
|
+
const inIdx = findOperatorOutsideQuotes(trimmed2, IN_OPERATOR_RE);
|
|
272152
|
+
const eqIdx = findOperatorOutsideQuotes(trimmed2, EQ_OPERATOR_RE);
|
|
272153
|
+
let operator;
|
|
272154
|
+
let opIdx;
|
|
272155
|
+
let opLen;
|
|
272156
|
+
if (inIdx !== -1 && (eqIdx === -1 || inIdx < eqIdx)) {
|
|
272157
|
+
operator = "in";
|
|
272158
|
+
opIdx = inIdx;
|
|
272159
|
+
opLen = 2;
|
|
272160
|
+
} else if (eqIdx !== -1) {
|
|
272161
|
+
operator = "=";
|
|
272162
|
+
opIdx = eqIdx;
|
|
272163
|
+
opLen = 1;
|
|
272164
|
+
} else {
|
|
272165
|
+
reject(sourceName, body, "malformed_body", "no `=` or `in` operator was found.");
|
|
272166
|
+
}
|
|
272167
|
+
let left = trimmed2.slice(0, opIdx).trim();
|
|
272168
|
+
let right = trimmed2.slice(opIdx + opLen).trim();
|
|
272169
|
+
if (GIVEN_REF_RE.test(left) && STRING_LITERAL_RE.test(right)) {
|
|
272170
|
+
if (operator === "in") {
|
|
272171
|
+
reject(sourceName, body, "reversed_in_operands", `\`${left} in ${right}\` reads $GIVEN in 'literal', which Malloy ` + `rejects as array-in-string membership. Write the literal ` + `first: \`${right} in ${left}\`.`);
|
|
272172
|
+
}
|
|
272173
|
+
[left, right] = [right, left];
|
|
272174
|
+
}
|
|
272175
|
+
const leftIsFieldPath = FIELD_PATH_RE.test(left);
|
|
272176
|
+
const leftIsLiteral = STRING_LITERAL_RE.test(left);
|
|
272177
|
+
if (!leftIsFieldPath && !leftIsLiteral) {
|
|
272178
|
+
reject(sourceName, body, "left_not_field_path", `\`${left}\` is neither a field path nor a string literal — one ` + "side must be a single column, a dotted join path, or a quoted " + "literal, and the other a `$GIVEN`.");
|
|
272179
|
+
}
|
|
272180
|
+
const givenMatch = GIVEN_REF_RE.exec(right);
|
|
272181
|
+
if (!givenMatch) {
|
|
272182
|
+
reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — a term compares against ` + "`$NAME`.");
|
|
272183
|
+
}
|
|
272184
|
+
const given = givenMatch[1];
|
|
272185
|
+
const declaredType = givenDeclaredTypes.get(given);
|
|
272186
|
+
if (declaredType !== undefined) {
|
|
272187
|
+
const isListType = declaredType === "array" || declaredType.endsWith("[]");
|
|
272188
|
+
if (operator === "=" && isListType) {
|
|
272189
|
+
reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a list-typed given — use \`in\`, not \`=\`.`);
|
|
272190
|
+
}
|
|
272191
|
+
if (operator === "in" && !isListType) {
|
|
272192
|
+
reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a scalar given — use \`=\`, not \`in\`.`);
|
|
272193
|
+
}
|
|
272194
|
+
}
|
|
272195
|
+
if (leftIsLiteral) {
|
|
272196
|
+
return { scope: "source_level", literal: left, given };
|
|
272197
|
+
}
|
|
272198
|
+
return {
|
|
272199
|
+
scope: "row_level",
|
|
272200
|
+
fieldPath: left,
|
|
272201
|
+
fieldPathSegments: splitFieldPathSegments(left),
|
|
272202
|
+
given,
|
|
272203
|
+
operator
|
|
272204
|
+
};
|
|
272205
|
+
}
|
|
272206
|
+
function assertAuthorizeGrammarTermsCoherent(sourceName, terms) {
|
|
272207
|
+
if (terms.length > 1 && terms.some(({ term }) => term.scope === "deny_all")) {
|
|
272208
|
+
rejectCoherence(sourceName, "deny_all_with_sibling", "an unconditional `false` deny-all cannot be combined with any " + "other authorize note — a deny-all admits nothing, so a " + "sibling note can never change what is served and its " + "presence is very likely a mistake.");
|
|
272209
|
+
}
|
|
272210
|
+
const byRoute = new Map;
|
|
272211
|
+
for (const { term, route } of terms) {
|
|
272212
|
+
if (term.scope === "deny_all")
|
|
272213
|
+
continue;
|
|
272214
|
+
const list = byRoute.get(route);
|
|
272215
|
+
if (list)
|
|
272216
|
+
list.push(term);
|
|
272217
|
+
else
|
|
272218
|
+
byRoute.set(route, [term]);
|
|
272219
|
+
}
|
|
272220
|
+
for (const allRouteTerms of byRoute.values()) {
|
|
272221
|
+
if (allRouteTerms.length > 1 && allRouteTerms.some((t) => t.scope === "admit_all")) {
|
|
272222
|
+
rejectCoherence(sourceName, "admit_all_with_sibling", "an unconditional `true` admit-all cannot be combined with " + "another `#(authorize)` note — the lock's notes AND " + "together, so `true and x` reduces to `x` and the admit-all " + "is dead text. To unlock a base while its rows stay " + "filtered, pair the `true` with `#(access_filter)` instead.");
|
|
272223
|
+
}
|
|
272224
|
+
const routeTerms = allRouteTerms.filter((t) => t.scope !== "admit_all" && t.scope !== "deny_all");
|
|
272225
|
+
const seenGivens = new Set;
|
|
272226
|
+
const seenFieldPaths = new Set;
|
|
272227
|
+
for (const t of routeTerms) {
|
|
272228
|
+
if (seenGivens.has(t.given)) {
|
|
272229
|
+
rejectCoherence(sourceName, "duplicate_given", `\`$${t.given}\` is used by more than one term — a given ` + "can back at most one term per route.");
|
|
272230
|
+
}
|
|
272231
|
+
seenGivens.add(t.given);
|
|
272232
|
+
if (t.scope === "row_level") {
|
|
272233
|
+
const canonicalPath = JSON.stringify(t.fieldPathSegments);
|
|
272234
|
+
if (seenFieldPaths.has(canonicalPath)) {
|
|
272235
|
+
rejectCoherence(sourceName, "duplicate_field_path", `\`${t.fieldPath}\` is used by more than one term.`);
|
|
272236
|
+
}
|
|
272237
|
+
seenFieldPaths.add(canonicalPath);
|
|
272238
|
+
}
|
|
272239
|
+
}
|
|
272240
|
+
}
|
|
272241
|
+
}
|
|
272242
|
+
function parseAuthorizeGrammarBody(sourceName, body, givenDeclaredTypes, route) {
|
|
272243
|
+
const trimmedBody = body.trim();
|
|
272244
|
+
if (trimmedBody.length === 0) {
|
|
272245
|
+
reject(sourceName, trimmedBody, "empty_body", "the expression body is empty.");
|
|
272246
|
+
}
|
|
272247
|
+
const sentinel = trimmedBody.toLowerCase();
|
|
272248
|
+
if (sentinel === "false" || sentinel === "true") {
|
|
272249
|
+
if (route !== AUTHORIZE_ROUTE) {
|
|
272250
|
+
reject(sourceName, trimmedBody, "sentinel_in_access_filter", `a bare \`${sentinel}\` is not allowed in \`#(access_filter)\` — ` + (sentinel === "false" ? "write `#(authorize) false`, which refuses the caller " + "rather than serving them zero rows." : "it filters nothing; delete the annotation, or write " + "`#(authorize) true` to re-open a locked base."));
|
|
272251
|
+
}
|
|
272252
|
+
return [{ scope: sentinel === "false" ? "deny_all" : "admit_all" }];
|
|
272253
|
+
}
|
|
272254
|
+
const { terms: rawTerms, compoundTokens } = splitTerms(trimmedBody);
|
|
272255
|
+
if (compoundTokens.length > 0) {
|
|
272256
|
+
reject(sourceName, trimmedBody, "compound_boolean", `a compound boolean (\`${compoundTokens[0]}\`) is not allowed — ` + "join terms with `and` only.");
|
|
272257
|
+
}
|
|
272258
|
+
const parsed = rawTerms.map((term) => parseTerm(sourceName, trimmedBody, term, givenDeclaredTypes));
|
|
272259
|
+
const ADMITTED_SCOPE = {
|
|
272260
|
+
[AUTHORIZE_ROUTE]: "source_level",
|
|
272261
|
+
[ACCESS_FILTER_ROUTE]: "row_level"
|
|
272262
|
+
};
|
|
272263
|
+
const admitted = ADMITTED_SCOPE[route];
|
|
272264
|
+
const offending = parsed.find((t) => t.scope !== admitted);
|
|
272265
|
+
if (offending) {
|
|
272266
|
+
const inLock = route === AUTHORIZE_ROUTE;
|
|
272267
|
+
reject(sourceName, trimmedBody, inLock ? "row_level_term_in_authorize" : "source_level_term_in_access_filter", inLock ? "a row-level term (field on the left) is not allowed in " + "`#(authorize)`, which decides whether the caller may reach " + "the source at all — move the term to `#(access_filter)`." : "a source-level term (literal on the left) is not allowed in " + "`#(access_filter)`, which decides which rows a caller may " + "see — move the term to `#(authorize)`, which refuses a " + "caller it does not admit rather than serving them zero rows.");
|
|
272268
|
+
}
|
|
272269
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, parsed.map((term) => ({ term, route })));
|
|
272270
|
+
return parsed;
|
|
272271
|
+
}
|
|
272272
|
+
function containsRetiredRouteTag(texts) {
|
|
272273
|
+
for (const text of texts) {
|
|
272274
|
+
const route = noteRoute2(text);
|
|
272275
|
+
if (route !== undefined && RETIRED_ROUTES.includes(route)) {
|
|
272276
|
+
return route;
|
|
272277
|
+
}
|
|
272278
|
+
}
|
|
272279
|
+
return;
|
|
272280
|
+
}
|
|
272281
|
+
function reachesRetiredRouteTagBelow(node, seen = new WeakSet, depth = 0) {
|
|
272282
|
+
if (depth > MAX_RETIRED_ROUTE_WALK_DEPTH) {
|
|
272283
|
+
throw new Error("retired-route IR walk exceeded max depth");
|
|
272284
|
+
}
|
|
272285
|
+
if (node === null || typeof node !== "object")
|
|
272286
|
+
return;
|
|
272287
|
+
if (seen.has(node))
|
|
272288
|
+
return;
|
|
272289
|
+
seen.add(node);
|
|
272290
|
+
if (Array.isArray(node)) {
|
|
272291
|
+
for (const item of node) {
|
|
272292
|
+
const found = reachesRetiredRouteTagBelow(item, seen, depth + 1);
|
|
272293
|
+
if (found)
|
|
272294
|
+
return found;
|
|
272295
|
+
}
|
|
272296
|
+
return;
|
|
272297
|
+
}
|
|
272298
|
+
const record = node;
|
|
272299
|
+
if (depth > 0) {
|
|
272300
|
+
for (const key of ["blockNotes", "notes"]) {
|
|
272301
|
+
const arr = record[key];
|
|
272302
|
+
if (!Array.isArray(arr))
|
|
272303
|
+
continue;
|
|
272304
|
+
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
272305
|
+
const found = containsRetiredRouteTag(texts);
|
|
272306
|
+
if (found)
|
|
272307
|
+
return found;
|
|
272308
|
+
}
|
|
272309
|
+
}
|
|
272310
|
+
for (const [key, value] of Object.entries(record)) {
|
|
272311
|
+
if (depth === 0 && key === "annotations")
|
|
272312
|
+
continue;
|
|
272313
|
+
const found = reachesRetiredRouteTagBelow(value, seen, depth + 1);
|
|
272314
|
+
if (found)
|
|
272315
|
+
return found;
|
|
272316
|
+
}
|
|
272317
|
+
return;
|
|
272318
|
+
}
|
|
272319
|
+
var AuthorizeGrammarError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", QUOTED_IDENT = "`[^`]+`", SEGMENT, FIELD_PATH_RE, GIVEN_REF_RE, STRING_LITERAL_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE, GRAMMAR_SUMMARY, IN_OPERATOR_RE, EQ_OPERATOR_RE, RETIRED_ROUTES, MAX_RETIRED_ROUTE_WALK_DEPTH = 200;
|
|
272320
|
+
var init_authorize_grammar = __esm(() => {
|
|
272321
|
+
init_errors();
|
|
272322
|
+
init_authorize_routes();
|
|
272323
|
+
AuthorizeGrammarError = class AuthorizeGrammarError extends ModelCompilationError {
|
|
272324
|
+
rejectionCause;
|
|
272325
|
+
constructor(rejectionCause, message) {
|
|
272326
|
+
super({ message });
|
|
272327
|
+
this.rejectionCause = rejectionCause;
|
|
272328
|
+
}
|
|
272329
|
+
};
|
|
272330
|
+
SEGMENT = `(?:${IDENT}|${QUOTED_IDENT})`;
|
|
272331
|
+
FIELD_PATH_RE = new RegExp(`^${SEGMENT}(?:\\.${SEGMENT})*$`);
|
|
272332
|
+
GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
|
|
272333
|
+
STRING_LITERAL_RE = /^(?:'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")$/;
|
|
272334
|
+
NEGATED_OPERATOR_RE = /!=/gu;
|
|
272335
|
+
COMPARISON_OPERATOR_RE = /(?:>=|<=|>|<)/gu;
|
|
272336
|
+
GRAMMAR_SUMMARY = "An authorize gate only accepts one or more `and`-joined terms, each " + "either `<column> <op> $GIVEN` (a single field or a dotted join " + "path on the left) or `'<literal>' in $GIVEN` (a literal on the " + "left), where <op> is fixed by the given's own arity: `in` for a " + "list-typed given, `=` for a scalar one.";
|
|
272337
|
+
IN_OPERATOR_RE = /\bin\b/giu;
|
|
272338
|
+
EQ_OPERATOR_RE = /=/gu;
|
|
272339
|
+
RETIRED_ROUTES = ["partition"];
|
|
272340
|
+
});
|
|
272341
|
+
|
|
271921
272342
|
// src/service/gate_dimension.ts
|
|
271922
272343
|
import {
|
|
271923
272344
|
isJoined,
|
|
@@ -271982,11 +272403,12 @@ function containsNegatedMembership(node, depth = 0) {
|
|
|
271982
272403
|
return containsNegatedMembership(n.e, depth + 1);
|
|
271983
272404
|
return false;
|
|
271984
272405
|
}
|
|
271985
|
-
function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, conditionExpr, modelDef, onWarning) {
|
|
272406
|
+
function validateSourceLineGateGivenUsage(sourceName, route, struct, refSummary, conditionExpr, modelDef, onWarning) {
|
|
272407
|
+
const gate = `#(${route}) gate`;
|
|
271986
272408
|
const expansion = expandRefSummaryGivenIds(struct, refSummary);
|
|
271987
272409
|
if (!expansion.ok) {
|
|
271988
272410
|
throw new ModelCompilationError({
|
|
271989
|
-
message:
|
|
272411
|
+
message: `The ${gate} on source "${sourceName}" references ` + `"${expansion.unresolvedPath}", which could not be resolved to a ` + `field this model can reach. An unresolvable reference is refused, ` + `not treated as referencing no given`
|
|
271990
272412
|
});
|
|
271991
272413
|
}
|
|
271992
272414
|
const givenIds = expansion.givenIds;
|
|
@@ -271994,15 +272416,15 @@ function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, condit
|
|
|
271994
272416
|
const given = modelDef.givens?.[id];
|
|
271995
272417
|
if (given && (given.default !== undefined || given.defaultText !== undefined)) {
|
|
271996
272418
|
throw new ModelCompilationError({
|
|
271997
|
-
message:
|
|
272419
|
+
message: `The ${gate} on source "${sourceName}" references ` + `\`$${given.name}\`, which is declared with a default. A ` + `caller who supplies no value for \`$${given.name}\` gets ` + `that default, which can admit what the gate was meant to ` + `exclude. Declare \`$${given.name}\` with no default`
|
|
271998
272420
|
});
|
|
271999
272421
|
}
|
|
272000
272422
|
}
|
|
272001
272423
|
if (givenIds.size === 0) {
|
|
272002
|
-
onWarning?.("source_line_gate_no_given_reference",
|
|
272424
|
+
onWarning?.("source_line_gate_no_given_reference", `The ${gate} on source "${sourceName}" references no given; it is ` + `a fixed predicate, not an access rule keyed on the caller`);
|
|
272003
272425
|
}
|
|
272004
272426
|
if (containsNegatedMembership(conditionExpr)) {
|
|
272005
|
-
onWarning?.("source_line_gate_negated_membership",
|
|
272427
|
+
onWarning?.("source_line_gate_negated_membership", `The ${gate} on source "${sourceName}" negates a membership test; ` + `an empty given then matches everything instead of nothing`);
|
|
272006
272428
|
}
|
|
272007
272429
|
}
|
|
272008
272430
|
var init_gate_dimension = __esm(() => {
|
|
@@ -272030,10 +272452,10 @@ function resolveDeclaredSource(struct, modelDef) {
|
|
|
272030
272452
|
}
|
|
272031
272453
|
return sawBrokenEntry ? { kind: "unresolvable" } : { kind: "none" };
|
|
272032
272454
|
}
|
|
272033
|
-
function ancestorGateExprs(struct, modelDef, seen = new Set) {
|
|
272455
|
+
function ancestorGateExprs(struct, modelDef, route, seen = new Set) {
|
|
272034
272456
|
let inherited = struct.annotations?.inherits;
|
|
272035
272457
|
for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
|
|
272036
|
-
const exprs2 =
|
|
272458
|
+
const exprs2 = collectAuthorizeExprsForRoute(ownLevelNoteTexts(inherited), route);
|
|
272037
272459
|
if (exprs2.length > 0)
|
|
272038
272460
|
return exprs2;
|
|
272039
272461
|
inherited = inherited.inherits;
|
|
@@ -272041,15 +272463,17 @@ function ancestorGateExprs(struct, modelDef, seen = new Set) {
|
|
|
272041
272463
|
if (inherited)
|
|
272042
272464
|
return ["false"];
|
|
272043
272465
|
seen.add(struct);
|
|
272044
|
-
if (seen.size > ANCESTOR_WALK_MAX_DEPTH)
|
|
272466
|
+
if (seen.size > ANCESTOR_WALK_MAX_DEPTH) {
|
|
272045
272467
|
return ["false"];
|
|
272468
|
+
}
|
|
272046
272469
|
const declared = resolveDeclaredSource(struct, modelDef);
|
|
272047
|
-
if (declared.kind === "unresolvable")
|
|
272470
|
+
if (declared.kind === "unresolvable") {
|
|
272048
272471
|
return ["false"];
|
|
272472
|
+
}
|
|
272049
272473
|
if (declared.kind === "none" || seen.has(declared.source))
|
|
272050
272474
|
return [];
|
|
272051
|
-
const exprs =
|
|
272052
|
-
return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, seen);
|
|
272475
|
+
const exprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(declared.source.annotations), route);
|
|
272476
|
+
return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, route, seen);
|
|
272053
272477
|
}
|
|
272054
272478
|
function resolveQuerySourceBase(struct, modelDef) {
|
|
272055
272479
|
const duck = struct;
|
|
@@ -272063,8 +272487,8 @@ function resolveCompositeResolvedBase(struct) {
|
|
|
272063
272487
|
const duck = struct;
|
|
272064
272488
|
return duck.type === "query_source" ? duck.query?.compositeResolvedSourceDef : undefined;
|
|
272065
272489
|
}
|
|
272066
|
-
function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
|
|
272067
|
-
const direct = ancestorGateExprs(struct, modelDef, new Set(seen));
|
|
272490
|
+
function effectiveAncestorGateExprs(struct, modelDef, route, seen = new Set) {
|
|
272491
|
+
const direct = ancestorGateExprs(struct, modelDef, route, new Set(seen));
|
|
272068
272492
|
if (direct.length > 0)
|
|
272069
272493
|
return [direct];
|
|
272070
272494
|
if (seen.has(struct))
|
|
@@ -272074,18 +272498,19 @@ function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
|
|
|
272074
272498
|
const base = resolveQuerySourceBase(struct, modelDef);
|
|
272075
272499
|
if (!base) {
|
|
272076
272500
|
const duck = struct;
|
|
272077
|
-
if (duck.type === "query_source")
|
|
272501
|
+
if (duck.type === "query_source") {
|
|
272078
272502
|
groups.push(["false"]);
|
|
272503
|
+
}
|
|
272079
272504
|
} else if (!seen.has(base)) {
|
|
272080
|
-
const ownExprs =
|
|
272081
|
-
groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, seen));
|
|
272505
|
+
const ownExprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(base.annotations), route);
|
|
272506
|
+
groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, route, seen));
|
|
272082
272507
|
}
|
|
272083
272508
|
const composite = resolveCompositeResolvedBase(struct);
|
|
272084
272509
|
if (composite && !seen.has(composite)) {
|
|
272085
272510
|
const parentOwnNotes = base ? ownLevelNotes(base.annotations) : [];
|
|
272086
272511
|
const compositeOwnNotes = ownLevelNotes(composite.annotations).filter((note) => !parentOwnNotes.includes(note));
|
|
272087
|
-
const compositeOwn =
|
|
272088
|
-
groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, seen));
|
|
272512
|
+
const compositeOwn = collectAuthorizeExprsForRoute(compositeOwnNotes.map((note) => note.text), route);
|
|
272513
|
+
groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, route, seen));
|
|
272089
272514
|
}
|
|
272090
272515
|
return groups;
|
|
272091
272516
|
}
|
|
@@ -272114,120 +272539,9 @@ var init_gate_registry_walk = __esm(() => {
|
|
|
272114
272539
|
init_authorize();
|
|
272115
272540
|
});
|
|
272116
272541
|
|
|
272117
|
-
// src/service/partition_annotation.ts
|
|
272118
|
-
import { payloadOf as payloadOf2, routeOf as routeOf2 } from "@malloydata/malloy";
|
|
272119
|
-
function noteRoute2(text) {
|
|
272120
|
-
return routeOf2({ value: text.trimStart() });
|
|
272121
|
-
}
|
|
272122
|
-
function containsPartitionAnnotationTag(texts) {
|
|
272123
|
-
return texts.some((text) => noteRoute2(text) === PARTITION_ROUTE);
|
|
272124
|
-
}
|
|
272125
|
-
function reachesPartitionTagBelow(node, seen = new WeakSet, depth = 0) {
|
|
272126
|
-
if (depth > MAX_PARTITION_IR_WALK_DEPTH) {
|
|
272127
|
-
throw new Error("partition-marker IR walk exceeded max depth");
|
|
272128
|
-
}
|
|
272129
|
-
if (node === null || typeof node !== "object")
|
|
272130
|
-
return false;
|
|
272131
|
-
if (seen.has(node))
|
|
272132
|
-
return false;
|
|
272133
|
-
seen.add(node);
|
|
272134
|
-
if (Array.isArray(node)) {
|
|
272135
|
-
return node.some((item) => reachesPartitionTagBelow(item, seen, depth + 1));
|
|
272136
|
-
}
|
|
272137
|
-
const record = node;
|
|
272138
|
-
if (depth > 0 && record.join !== undefined)
|
|
272139
|
-
return false;
|
|
272140
|
-
if (depth > 0) {
|
|
272141
|
-
for (const key of ["blockNotes", "notes"]) {
|
|
272142
|
-
const arr = record[key];
|
|
272143
|
-
if (!Array.isArray(arr))
|
|
272144
|
-
continue;
|
|
272145
|
-
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
272146
|
-
if (containsPartitionAnnotationTag(texts))
|
|
272147
|
-
return true;
|
|
272148
|
-
}
|
|
272149
|
-
}
|
|
272150
|
-
return Object.entries(record).some(([key, value]) => depth === 0 && key === "annotations" ? false : reachesPartitionTagBelow(value, seen, depth + 1));
|
|
272151
|
-
}
|
|
272152
|
-
function notePayload2(text) {
|
|
272153
|
-
return payloadOf2({ value: text.trimStart() }) ?? "";
|
|
272154
|
-
}
|
|
272155
|
-
function rejectionMessage(sourceName, body, detail) {
|
|
272156
|
-
return `Source "${sourceName}" declares \`#(partition) ${body}\`: ${detail} ` + `#(partition) only accepts \`<column> = $GIVEN\`, where <column> is a ` + `single field or a dotted join path.`;
|
|
272157
|
-
}
|
|
272158
|
-
function reject(sourceName, body, cause, detail) {
|
|
272159
|
-
throw new PartitionAnnotationError(cause, rejectionMessage(sourceName, body, detail));
|
|
272160
|
-
}
|
|
272161
|
-
function parsePartitionAnnotation(sourceName, annotationText) {
|
|
272162
|
-
if (noteRoute2(annotationText) !== PARTITION_ROUTE)
|
|
272163
|
-
return null;
|
|
272164
|
-
const body = notePayload2(annotationText).trim();
|
|
272165
|
-
if (body.length === 0) {
|
|
272166
|
-
reject(sourceName, body, "empty_body", "the expression body is empty.");
|
|
272167
|
-
}
|
|
272168
|
-
if (COMPOUND_BOOLEAN_RE.test(body)) {
|
|
272169
|
-
reject(sourceName, body, "compound_boolean", "a compound boolean (`and`/`or`/`not`) is not allowed — declare one " + "`#(partition)` marker per column.");
|
|
272170
|
-
}
|
|
272171
|
-
if (IN_OPERATOR_RE.test(body)) {
|
|
272172
|
-
reject(sourceName, body, "in_operator", "the `in` operator is not allowed.");
|
|
272173
|
-
}
|
|
272174
|
-
if (NEGATED_OPERATOR_RE.test(body)) {
|
|
272175
|
-
reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
|
|
272176
|
-
}
|
|
272177
|
-
if (COMPARISON_OPERATOR_RE.test(body)) {
|
|
272178
|
-
reject(sourceName, body, "comparison_operator", "only `=` is allowed, not `<`/`>`/`<=`/`>=`.");
|
|
272179
|
-
}
|
|
272180
|
-
const eq = body.indexOf("=");
|
|
272181
|
-
if (eq === -1) {
|
|
272182
|
-
reject(sourceName, body, "malformed_body", "no `=` was found.");
|
|
272183
|
-
}
|
|
272184
|
-
const left = body.slice(0, eq).trim();
|
|
272185
|
-
const right = body.slice(eq + 1).trim();
|
|
272186
|
-
if (!FIELD_PATH_RE.test(left)) {
|
|
272187
|
-
reject(sourceName, body, "left_not_field_path", `\`${left}\` is not a field path — the left side must be a single ` + "column or a dotted join path, not an expression.");
|
|
272188
|
-
}
|
|
272189
|
-
const givenMatch = GIVEN_REF_RE.exec(right);
|
|
272190
|
-
if (!givenMatch) {
|
|
272191
|
-
reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — the right side must be ` + "`$NAME`.");
|
|
272192
|
-
}
|
|
272193
|
-
return { column: left, given: givenMatch[1] };
|
|
272194
|
-
}
|
|
272195
|
-
function collectPartitionPairs(sourceName, annotationTexts2) {
|
|
272196
|
-
const pairs = [];
|
|
272197
|
-
const seenGivens = new Map;
|
|
272198
|
-
for (const text of annotationTexts2) {
|
|
272199
|
-
const pair = parsePartitionAnnotation(sourceName, text);
|
|
272200
|
-
if (pair === null)
|
|
272201
|
-
continue;
|
|
272202
|
-
const priorColumn = seenGivens.get(pair.given);
|
|
272203
|
-
if (priorColumn !== undefined) {
|
|
272204
|
-
throw new PartitionAnnotationError("duplicate_given", `Source "${sourceName}" declares \`#(partition)\` on both ` + `\`${priorColumn}\` and \`${pair.column}\` for the same given ` + `\`$${pair.given}\` — a given can back at most one partition ` + `column per source.`);
|
|
272205
|
-
}
|
|
272206
|
-
seenGivens.set(pair.given, pair.column);
|
|
272207
|
-
pairs.push(pair);
|
|
272208
|
-
}
|
|
272209
|
-
return pairs;
|
|
272210
|
-
}
|
|
272211
|
-
var PARTITION_ROUTE = "partition", MAX_PARTITION_IR_WALK_DEPTH = 200, PartitionAnnotationError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", FIELD_PATH_RE, GIVEN_REF_RE, COMPOUND_BOOLEAN_RE, IN_OPERATOR_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE;
|
|
272212
|
-
var init_partition_annotation = __esm(() => {
|
|
272213
|
-
init_errors();
|
|
272214
|
-
PartitionAnnotationError = class PartitionAnnotationError extends ModelCompilationError {
|
|
272215
|
-
rejectionCause;
|
|
272216
|
-
constructor(rejectionCause, message) {
|
|
272217
|
-
super({ message });
|
|
272218
|
-
this.rejectionCause = rejectionCause;
|
|
272219
|
-
}
|
|
272220
|
-
};
|
|
272221
|
-
FIELD_PATH_RE = new RegExp(`^${IDENT}(?:\\.${IDENT})*$`);
|
|
272222
|
-
GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
|
|
272223
|
-
COMPOUND_BOOLEAN_RE = /\b(and|or|not)\b/i;
|
|
272224
|
-
IN_OPERATOR_RE = /\bin\b/i;
|
|
272225
|
-
NEGATED_OPERATOR_RE = /!=/;
|
|
272226
|
-
COMPARISON_OPERATOR_RE = /(>=|<=|>|<)/;
|
|
272227
|
-
});
|
|
272228
|
-
|
|
272229
272542
|
// src/service/gate_classification.ts
|
|
272230
272543
|
import {
|
|
272544
|
+
isJoined as isJoined2,
|
|
272231
272545
|
isSourceDef as isSourceDef5
|
|
272232
272546
|
} from "@malloydata/malloy";
|
|
272233
272547
|
function createGateClassificationDeps(givens, modelPath) {
|
|
@@ -272237,30 +272551,41 @@ function createGateClassificationDeps(givens, modelPath) {
|
|
|
272237
272551
|
modelPath
|
|
272238
272552
|
};
|
|
272239
272553
|
}
|
|
272240
|
-
function gateExprsForOwnAnnotations(struct, modelDef, excludeNotes = []) {
|
|
272554
|
+
function gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes = []) {
|
|
272241
272555
|
const ownNotes = ownLevelNotes(struct.annotations).filter((note) => !excludeNotes.includes(note));
|
|
272242
272556
|
try {
|
|
272243
|
-
const own =
|
|
272557
|
+
const own = collectAuthorizeExprsForRoute(ownNotes.map((note) => note.text), route);
|
|
272244
272558
|
if (own.length > 0) {
|
|
272245
272559
|
return { exprs: own, fromAncestor: false };
|
|
272246
272560
|
}
|
|
272247
|
-
const ancestor = ancestorGateExprs(struct, modelDef);
|
|
272561
|
+
const ancestor = ancestorGateExprs(struct, modelDef, route);
|
|
272248
272562
|
return { exprs: ancestor, fromAncestor: ancestor.length > 0 };
|
|
272249
272563
|
} catch {
|
|
272250
|
-
return {
|
|
272564
|
+
return {
|
|
272565
|
+
exprs: ["false"],
|
|
272566
|
+
fromAncestor: false
|
|
272567
|
+
};
|
|
272251
272568
|
}
|
|
272252
272569
|
}
|
|
272253
272570
|
function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate = false, entryPointStruct = struct, excludeNotes = []) {
|
|
272571
|
+
const results = [];
|
|
272572
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
272573
|
+
results.push(...collectEntryPointGatesForRoute(struct, modelDef, route, new Set(seen), treatAsOwnGate, entryPointStruct, excludeNotes));
|
|
272574
|
+
}
|
|
272575
|
+
return results;
|
|
272576
|
+
}
|
|
272577
|
+
function collectEntryPointGatesForRoute(struct, modelDef, route, seen, treatAsOwnGate, entryPointStruct, excludeNotes) {
|
|
272254
272578
|
if (!struct || !modelDef || seen.has(struct))
|
|
272255
272579
|
return [];
|
|
272256
272580
|
seen.add(struct);
|
|
272257
272581
|
const results = [];
|
|
272258
272582
|
const label = struct.as ?? struct.name;
|
|
272259
|
-
const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, modelDef, excludeNotes);
|
|
272583
|
+
const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes);
|
|
272260
272584
|
if (ownExprs.length > 0) {
|
|
272261
272585
|
results.push({
|
|
272262
272586
|
label,
|
|
272263
272587
|
exprs: ownExprs,
|
|
272588
|
+
route,
|
|
272264
272589
|
selfContained: fromAncestor || !treatAsOwnGate,
|
|
272265
272590
|
struct: entryPointStruct
|
|
272266
272591
|
});
|
|
@@ -272269,17 +272594,18 @@ function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate
|
|
|
272269
272594
|
if (duck.type === "query_source") {
|
|
272270
272595
|
const base = resolveQuerySourceBase(struct, modelDef);
|
|
272271
272596
|
if (base) {
|
|
272272
|
-
results.push(...
|
|
272597
|
+
results.push(...collectEntryPointGatesForRoute(base, modelDef, route, seen, false, entryPointStruct, []));
|
|
272273
272598
|
} else {
|
|
272274
272599
|
results.push({
|
|
272275
272600
|
label,
|
|
272276
272601
|
exprs: ["false"],
|
|
272602
|
+
route,
|
|
272277
272603
|
selfContained: true
|
|
272278
272604
|
});
|
|
272279
272605
|
}
|
|
272280
272606
|
const resolved = duck.query?.compositeResolvedSourceDef;
|
|
272281
272607
|
if (resolved) {
|
|
272282
|
-
results.push(...
|
|
272608
|
+
results.push(...collectEntryPointGatesForRoute(resolved, modelDef, route, seen, false, entryPointStruct, base ? ownLevelNotes(base.annotations) : []));
|
|
272283
272609
|
}
|
|
272284
272610
|
}
|
|
272285
272611
|
return results;
|
|
@@ -272296,7 +272622,7 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272296
272622
|
return { shape: "rejected" };
|
|
272297
272623
|
}
|
|
272298
272624
|
const filterText = gateFilterText(entry.exprs);
|
|
272299
|
-
const cacheKey = `${graftScope.cacheScope}\x00${graftTarget}\x00${filterText}`;
|
|
272625
|
+
const cacheKey = `${graftScope.cacheScope}\x00${entry.route}\x00${graftTarget}\x00${filterText}`;
|
|
272300
272626
|
let cached2 = deps.gateShapeCache.get(cacheKey);
|
|
272301
272627
|
if (!cached2) {
|
|
272302
272628
|
let condition;
|
|
@@ -272319,7 +272645,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272319
272645
|
detail: "this entry's lifted condition carries no usable expression"
|
|
272320
272646
|
};
|
|
272321
272647
|
} else if (isBareFalseLiteral(condition.e)) {
|
|
272322
|
-
classification = {
|
|
272648
|
+
classification = {
|
|
272649
|
+
shape: "row_level",
|
|
272650
|
+
givenNames: [],
|
|
272651
|
+
givenNamesById: new Map
|
|
272652
|
+
};
|
|
272323
272653
|
} else {
|
|
272324
272654
|
const targetStruct = graftScope.modelDef.contents[graftTarget];
|
|
272325
272655
|
if (!isSourceDef5(targetStruct)) {
|
|
@@ -272337,7 +272667,13 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272337
272667
|
detail: `this gate references \`${expansion.unresolvedPath}\`, which could not be resolved on the graft target`
|
|
272338
272668
|
};
|
|
272339
272669
|
} else {
|
|
272340
|
-
const
|
|
272670
|
+
const givenNamesById = new Map;
|
|
272671
|
+
for (const id of expansion.givenIds) {
|
|
272672
|
+
const name = graftScope.modelDef.givens?.[id]?.name;
|
|
272673
|
+
if (name)
|
|
272674
|
+
givenNamesById.set(id, name);
|
|
272675
|
+
}
|
|
272676
|
+
const givenNames = Array.from(givenNamesById.values());
|
|
272341
272677
|
const literalNames = referencedGivenNames(filterText);
|
|
272342
272678
|
const accountedFor = new Set(givenNames);
|
|
272343
272679
|
const unaccounted = literalNames.filter((name) => !accountedFor.has(name));
|
|
@@ -272348,7 +272684,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272348
272684
|
detail: "this gate references a given id that does not resolve to a name on this model"
|
|
272349
272685
|
};
|
|
272350
272686
|
} else {
|
|
272351
|
-
classification = {
|
|
272687
|
+
classification = {
|
|
272688
|
+
shape: "row_level",
|
|
272689
|
+
givenNames,
|
|
272690
|
+
givenNamesById
|
|
272691
|
+
};
|
|
272352
272692
|
}
|
|
272353
272693
|
}
|
|
272354
272694
|
}
|
|
@@ -272374,6 +272714,14 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272374
272714
|
if (cached2.classification.shape === "rejected") {
|
|
272375
272715
|
return { shape: "rejected", cause: cached2.classification.cause };
|
|
272376
272716
|
}
|
|
272717
|
+
if (entry.route === AUTHORIZE_ROUTE) {
|
|
272718
|
+
return {
|
|
272719
|
+
shape: "lock",
|
|
272720
|
+
condition: cached2.condition,
|
|
272721
|
+
givenNames: cached2.classification.givenNames,
|
|
272722
|
+
givenNamesById: cached2.classification.givenNamesById
|
|
272723
|
+
};
|
|
272724
|
+
}
|
|
272377
272725
|
return {
|
|
272378
272726
|
shape: "row_level",
|
|
272379
272727
|
graftTarget,
|
|
@@ -272456,142 +272804,120 @@ function isBareFalseLiteral(expr) {
|
|
|
272456
272804
|
function computeGivenDeclaredTypes(givens) {
|
|
272457
272805
|
return new Map((givens ?? []).filter((g) => g.name != null && g.type != null).map((g) => [g.name, g.type]));
|
|
272458
272806
|
}
|
|
272459
|
-
function
|
|
272460
|
-
if (!struct || !modelDef || seen.has(struct))
|
|
272461
|
-
return [];
|
|
272462
|
-
seen.add(struct);
|
|
272463
|
-
const label = struct.as ?? struct.name;
|
|
272464
|
-
const own = collectPartitionPairs(label, ownLevelNotes(struct.annotations).map((note) => note.text));
|
|
272465
|
-
if (own.length > 0)
|
|
272466
|
-
return own;
|
|
272467
|
-
let inherited = struct.annotations?.inherits;
|
|
272468
|
-
for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
|
|
272469
|
-
const pairs = collectPartitionPairs(label, ownLevelNoteTexts(inherited));
|
|
272470
|
-
if (pairs.length > 0)
|
|
272471
|
-
return pairs;
|
|
272472
|
-
inherited = inherited.inherits;
|
|
272473
|
-
}
|
|
272474
|
-
if (inherited)
|
|
272475
|
-
throw unresolvableAncestry(label);
|
|
272476
|
-
const declared = resolveDeclaredSource(struct, modelDef);
|
|
272477
|
-
if (declared.kind === "unresolvable")
|
|
272478
|
-
throw unresolvableAncestry(label);
|
|
272479
|
-
if (declared.kind === "resolved") {
|
|
272480
|
-
const pairs = resolveEntryPointPartitions(declared.source, modelDef, seen);
|
|
272481
|
-
if (pairs.length > 0)
|
|
272482
|
-
return pairs;
|
|
272483
|
-
}
|
|
272484
|
-
const queryBase = resolveQuerySourceBase(struct, modelDef);
|
|
272485
|
-
if (queryBase)
|
|
272486
|
-
return resolveEntryPointPartitions(queryBase, modelDef, seen);
|
|
272487
|
-
return [];
|
|
272488
|
-
}
|
|
272489
|
-
function unresolvableAncestry(label) {
|
|
272490
|
-
return new PartitionAnnotationError("ancestry_unresolvable", `Could not resolve whether source "${label}" carries a ` + `\`#(partition)\` marker: the derivation chain it inherits from ` + `could not be read. Denying rather than serving unfiltered.`);
|
|
272491
|
-
}
|
|
272492
|
-
function assertPartitionAnnotationsValid(modelDef) {
|
|
272807
|
+
function assertAuthorizeGrammarValid(modelDef, authorizeMap, authorizeOwnNotes, givenDeclaredTypes, onAdmitAllGate, attributedAuthorizeOwnNotes) {
|
|
272493
272808
|
if (!modelDef)
|
|
272494
272809
|
return;
|
|
272495
|
-
for (const [
|
|
272496
|
-
|
|
272497
|
-
|
|
272498
|
-
const
|
|
272499
|
-
|
|
272500
|
-
|
|
272501
|
-
|
|
272502
|
-
|
|
272503
|
-
|
|
272504
|
-
|
|
272505
|
-
|
|
272810
|
+
for (const [sourceName, groups] of authorizeMap) {
|
|
272811
|
+
const struct = modelDef.contents[sourceName];
|
|
272812
|
+
const ownGroupTerms = [];
|
|
272813
|
+
const declaredAdmitAllRoutes = [];
|
|
272814
|
+
for (const { route, exprs } of groups) {
|
|
272815
|
+
const isOwn = (authorizeOwnNotes.get(sourceName)?.get(route)?.length ?? 0) > 0;
|
|
272816
|
+
const filteredExprs = exprs.filter((expr) => (isOwn || expr !== "false") && !isLegacyQuotedPayload(expr));
|
|
272817
|
+
const groupTerms = [];
|
|
272818
|
+
for (const expr of filteredExprs) {
|
|
272819
|
+
const terms = parseAuthorizeGrammarBody(sourceName, expr, givenDeclaredTypes, route);
|
|
272820
|
+
for (const term of terms) {
|
|
272821
|
+
groupTerms.push({ term, route });
|
|
272822
|
+
if (term.scope === "admit_all" && (attributedAuthorizeOwnNotes?.get(sourceName)?.get(route)?.length ?? 0) > 0) {
|
|
272823
|
+
declaredAdmitAllRoutes.push(route);
|
|
272824
|
+
}
|
|
272825
|
+
if (term.scope === "row_level" && isSourceDef5(struct)) {
|
|
272826
|
+
assertNoFanoutFieldPath(sourceName, struct, term.fieldPath, term.fieldPathSegments);
|
|
272827
|
+
}
|
|
272506
272828
|
}
|
|
272507
|
-
continue;
|
|
272508
272829
|
}
|
|
272509
|
-
|
|
272510
|
-
|
|
272830
|
+
if (isOwn) {
|
|
272831
|
+
ownGroupTerms.push(...groupTerms);
|
|
272832
|
+
} else {
|
|
272833
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, groupTerms);
|
|
272834
|
+
}
|
|
272511
272835
|
}
|
|
272512
|
-
if (
|
|
272513
|
-
|
|
272836
|
+
if (ownGroupTerms.length > 0) {
|
|
272837
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, ownGroupTerms);
|
|
272514
272838
|
}
|
|
272515
|
-
const
|
|
272516
|
-
|
|
272517
|
-
throw new PartitionAnnotationError("partitioned_composite", `Source "${label}" is a composite source (\`compose(...)\`) with ` + `member "${marked}", which declares \`#(partition)\`. MEASURED: ` + `querying the composite reads EVERY partition of that member — ` + `the graft lands on the composite's own struct, not the ` + `resolved member branch, so no filter reaches the executed ` + `query. Remove the composite, or drop \`#(partition)\` from ` + `"${marked}" and expose it as its own source.`);
|
|
272839
|
+
for (const route of declaredAdmitAllRoutes) {
|
|
272840
|
+
onAdmitAllGate?.(sourceName, route);
|
|
272518
272841
|
}
|
|
272519
272842
|
}
|
|
272520
272843
|
}
|
|
272521
|
-
function
|
|
272522
|
-
|
|
272523
|
-
|
|
272524
|
-
|
|
272525
|
-
|
|
272526
|
-
|
|
272527
|
-
|
|
272528
|
-
|
|
272529
|
-
|
|
272530
|
-
if (!reachesBelow)
|
|
272531
|
-
return;
|
|
272532
|
-
throw new PartitionAnnotationError("marker_unreachable", `Source "${label}" carries a \`#(partition)\` marker that is not ` + `declared on the source itself — it sits on a field, a view, or an ` + `inline \`compose(...)\` inside it. Nothing would filter this ` + `source: every caller would read every partition. Move the marker ` + `onto the \`source:\` declaration.`);
|
|
272533
|
-
}
|
|
272534
|
-
function partitionedMemberLabel(composite, modelDef, seen = new Set) {
|
|
272535
|
-
if (seen.has(composite))
|
|
272536
|
-
return;
|
|
272537
|
-
seen.add(composite);
|
|
272538
|
-
for (const member of composite.sources ?? []) {
|
|
272539
|
-
const label = member.as ?? member.name;
|
|
272540
|
-
if (resolveEntryPointPartitions(member, modelDef).length > 0) {
|
|
272541
|
-
return label;
|
|
272844
|
+
function fanoutSegment(struct, path9) {
|
|
272845
|
+
let current = struct;
|
|
272846
|
+
for (let i = 0;i < path9.length - 1; i++) {
|
|
272847
|
+
const seg = path9[i];
|
|
272848
|
+
const field = (current.fields ?? []).find((f) => (f.as ?? f.name) === seg);
|
|
272849
|
+
if (!field)
|
|
272850
|
+
return { segment: seg, reason: "unresolved" };
|
|
272851
|
+
if (isJoined2(field) && field.join !== "one") {
|
|
272852
|
+
return { segment: seg, reason: "fanout" };
|
|
272542
272853
|
}
|
|
272543
|
-
if (
|
|
272544
|
-
|
|
272545
|
-
if (nested !== undefined)
|
|
272546
|
-
return nested;
|
|
272854
|
+
if (!isSourceDef5(field) && field.type !== "record" && field.type !== "array") {
|
|
272855
|
+
return { segment: seg, reason: "unresolved" };
|
|
272547
272856
|
}
|
|
272857
|
+
current = field;
|
|
272548
272858
|
}
|
|
272549
272859
|
return;
|
|
272550
272860
|
}
|
|
272551
|
-
|
|
272552
|
-
|
|
272553
|
-
|
|
272554
|
-
|
|
272555
|
-
|
|
272556
|
-
|
|
272557
|
-
|
|
272558
|
-
|
|
272559
|
-
|
|
272560
|
-
|
|
272561
|
-
|
|
272562
|
-
const
|
|
272563
|
-
if (
|
|
272564
|
-
|
|
272565
|
-
modelPath: deps.modelPath,
|
|
272566
|
-
label
|
|
272567
|
-
});
|
|
272568
|
-
throw new Error(`partition on "${label}" resolved to no graft target`);
|
|
272861
|
+
function assertNoFanoutFieldPath(sourceName, struct, fieldPath, fieldPathSegments) {
|
|
272862
|
+
const finding = fanoutSegment(struct, fieldPathSegments);
|
|
272863
|
+
if (finding === undefined)
|
|
272864
|
+
return;
|
|
272865
|
+
const detail = finding.reason === "fanout" ? `which reaches through \`${finding.segment}\`, a fan-out join ` + `(\`join_many\`/\`join_cross\`, or a repeated record) — the ` + `entry row has many values for what follows, so there is no ` + `single key to compare against a given. Gate on a field reached ` + `only through \`join_one\` joins.` : `whose segment \`${finding.segment}\` does not resolve against ` + `this source — refused rather than assumed safe.`;
|
|
272866
|
+
throw new AuthorizeGrammarError("fanout_path", `Source "${sourceName}" declares an authorize gate on \`${fieldPath}\`, ${detail}`);
|
|
272867
|
+
}
|
|
272868
|
+
function collectRetiredRouteMarkers(modelDef) {
|
|
272869
|
+
const found = [];
|
|
272870
|
+
if (!modelDef)
|
|
272871
|
+
return found;
|
|
272872
|
+
const fileMatch = containsRetiredRouteTag(ownLevelNoteTexts(modelAnnotations(modelDef)));
|
|
272873
|
+
if (fileMatch) {
|
|
272874
|
+
found.push(`the model itself declares a retired \`##(${fileMatch})\` annotation`);
|
|
272569
272875
|
}
|
|
272570
|
-
const entries
|
|
272571
|
-
|
|
272572
|
-
if (!
|
|
272573
|
-
|
|
272574
|
-
|
|
272876
|
+
for (const [key, obj] of Object.entries(modelDef.contents)) {
|
|
272877
|
+
const isQuery = obj.type === "query";
|
|
272878
|
+
if (!isSourceDef5(obj) && !isQuery)
|
|
272879
|
+
continue;
|
|
272880
|
+
const label = obj.as ?? obj.name ?? key;
|
|
272881
|
+
const noun = isQuery ? "query" : "source";
|
|
272882
|
+
const ownMatch = containsRetiredRouteTag(annotationTexts(obj.annotations) ?? []);
|
|
272883
|
+
if (ownMatch) {
|
|
272884
|
+
found.push(`${noun} "${label}" declares a retired \`#(${ownMatch})\` annotation`);
|
|
272885
|
+
continue;
|
|
272886
|
+
}
|
|
272887
|
+
if (isQuery) {
|
|
272888
|
+
continue;
|
|
272889
|
+
}
|
|
272890
|
+
let belowMatch;
|
|
272891
|
+
try {
|
|
272892
|
+
belowMatch = reachesRetiredRouteTagBelow(obj);
|
|
272893
|
+
} catch {
|
|
272894
|
+
belowMatch = RETIRED_ROUTES[0];
|
|
272895
|
+
}
|
|
272896
|
+
if (belowMatch) {
|
|
272897
|
+
found.push(`source "${label}" carries a retired \`#(${belowMatch})\` ` + `annotation somewhere in its IR (not on the source line itself)`);
|
|
272575
272898
|
}
|
|
272576
|
-
const filterText = `${pair.column} = $${pair.given}`;
|
|
272577
|
-
const condition = await liftGateCondition(graftTarget, filterText, graftScope.materializer);
|
|
272578
|
-
entries.push({
|
|
272579
|
-
label,
|
|
272580
|
-
graftTarget,
|
|
272581
|
-
filterText,
|
|
272582
|
-
condition,
|
|
272583
|
-
givenNames: [pair.given]
|
|
272584
|
-
});
|
|
272585
272899
|
}
|
|
272586
|
-
return
|
|
272900
|
+
return found;
|
|
272901
|
+
}
|
|
272902
|
+
function assertNoRetiredRouteMarkers(found) {
|
|
272903
|
+
if (found.length === 0)
|
|
272904
|
+
return;
|
|
272905
|
+
throw new ModelCompilationError({
|
|
272906
|
+
message: `These sources carry a marker from a retired annotation route:
|
|
272907
|
+
` + `${found.map((f) => ` - ${f}`).join(`
|
|
272908
|
+
`)}
|
|
272909
|
+
` + `The route is no longer enforced by anything, so Malloy would ` + `otherwise parse it, load the model clean, and serve every row as ` + `if the marker were never written. Remove the annotation.`
|
|
272910
|
+
});
|
|
272587
272911
|
}
|
|
272588
272912
|
var init_gate_classification = __esm(() => {
|
|
272913
|
+
init_errors();
|
|
272589
272914
|
init_logger();
|
|
272590
272915
|
init_annotations();
|
|
272591
272916
|
init_authorize();
|
|
272917
|
+
init_authorize_grammar();
|
|
272918
|
+
init_authorize_routes();
|
|
272592
272919
|
init_gate_dimension();
|
|
272593
272920
|
init_gate_registry_walk();
|
|
272594
|
-
init_partition_annotation();
|
|
272595
272921
|
});
|
|
272596
272922
|
|
|
272597
272923
|
// src/service/persist_dynamic_terms.ts
|
|
@@ -273061,14 +273387,7 @@ function assertMaterializationEligible(persistSource, annotationFields) {
|
|
|
273061
273387
|
recordEligibilityRefused("authorize");
|
|
273062
273388
|
throw new MaterializationEligibilityError({
|
|
273063
273389
|
reason: "authorize",
|
|
273064
|
-
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by
|
|
273065
|
-
});
|
|
273066
|
-
}
|
|
273067
|
-
if (referencesPartition(persistSource)) {
|
|
273068
|
-
recordEligibilityRefused("partition");
|
|
273069
|
-
throw new MaterializationEligibilityError({
|
|
273070
|
-
reason: "partition",
|
|
273071
|
-
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it declares a #(partition) marker. A partition ` + `filter binds a given at query time (the same mechanism as ` + `row-level access control), so a materialized-once table served ` + `frozen would leak every partition's rows to every caller. This ` + `is refused for safety. Serve this source live (drop 'storage=').`
|
|
273390
|
+
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by a gate — #(authorize) or ` + `#(access_filter), its own or a joined source's. Both are evaluated ` + `per request; a materialized-once table served frozen carries no ` + `gate, so it would be served to everyone, bypassing authorization. ` + `This is refused for safety. Serve this source live (drop ` + `'storage=').`
|
|
273072
273391
|
});
|
|
273073
273392
|
}
|
|
273074
273393
|
const partition = resolvePartitionColumns(persistSource, annotationFields);
|
|
@@ -273104,15 +273423,6 @@ function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = per
|
|
|
273104
273423
|
message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${partition.detail}.`
|
|
273105
273424
|
});
|
|
273106
273425
|
}
|
|
273107
|
-
if (referencesPartition(persistSource)) {
|
|
273108
|
-
recordEligibilityRefused("partition");
|
|
273109
|
-
const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
|
|
273110
|
-
const gated2 = origin2 === "preaggregate" ? `the source '${sourceName}' rolls up declares` : `it declares`;
|
|
273111
|
-
throw new MaterializationEligibilityError({
|
|
273112
|
-
reason: "partition",
|
|
273113
|
-
message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${gated2} a #(partition) marker. A partition filter binds a ` + `given at query time; this pass cannot prove the persisted ` + `artifact's read path still applies it, so this is refused for ` + `safety. Move the marker to a source that is not materialized, ` + `or stop persisting this one.`
|
|
273114
|
-
});
|
|
273115
|
-
}
|
|
273116
273426
|
if (buildSubstitutesAGiven(persistSource)) {
|
|
273117
273427
|
if (origin2 === "preaggregate") {
|
|
273118
273428
|
recordEligibilityRefused("preaggregate_over_dynamic_source");
|
|
@@ -273163,44 +273473,6 @@ function unboundParameterNames(persistSource) {
|
|
|
273163
273473
|
}
|
|
273164
273474
|
return unbound;
|
|
273165
273475
|
}
|
|
273166
|
-
function referencesPartition(persistSource) {
|
|
273167
|
-
try {
|
|
273168
|
-
return walkForPartition(persistSource._sourceDef, new WeakSet, 0);
|
|
273169
|
-
} catch {
|
|
273170
|
-
return true;
|
|
273171
|
-
}
|
|
273172
|
-
}
|
|
273173
|
-
function walkForPartition(node, seen, depth) {
|
|
273174
|
-
if (depth > MAX_GIVEN_WALK_DEPTH) {
|
|
273175
|
-
throw new Error("partition-usage walk exceeded max depth");
|
|
273176
|
-
}
|
|
273177
|
-
if (node === null || typeof node !== "object")
|
|
273178
|
-
return false;
|
|
273179
|
-
if (seen.has(node))
|
|
273180
|
-
return false;
|
|
273181
|
-
seen.add(node);
|
|
273182
|
-
if (Array.isArray(node)) {
|
|
273183
|
-
for (const item of node) {
|
|
273184
|
-
if (walkForPartition(item, seen, depth + 1))
|
|
273185
|
-
return true;
|
|
273186
|
-
}
|
|
273187
|
-
return false;
|
|
273188
|
-
}
|
|
273189
|
-
const record = node;
|
|
273190
|
-
for (const key of ["blockNotes", "notes"]) {
|
|
273191
|
-
const arr = record[key];
|
|
273192
|
-
if (!Array.isArray(arr))
|
|
273193
|
-
continue;
|
|
273194
|
-
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
273195
|
-
if (containsPartitionAnnotationTag(texts))
|
|
273196
|
-
return true;
|
|
273197
|
-
}
|
|
273198
|
-
for (const value of Object.values(record)) {
|
|
273199
|
-
if (walkForPartition(value, seen, depth + 1))
|
|
273200
|
-
return true;
|
|
273201
|
-
}
|
|
273202
|
-
return false;
|
|
273203
|
-
}
|
|
273204
273476
|
function referencesAuthorize(persistSource) {
|
|
273205
273477
|
try {
|
|
273206
273478
|
return walkForAuthorize(persistSource._sourceDef, new WeakSet, 0);
|
|
@@ -273302,7 +273574,6 @@ var init_materialization_eligibility = __esm(() => {
|
|
|
273302
273574
|
init_materialization_metrics();
|
|
273303
273575
|
init_authorize();
|
|
273304
273576
|
init_build_plan();
|
|
273305
|
-
init_partition_annotation();
|
|
273306
273577
|
init_persist_dynamic_terms();
|
|
273307
273578
|
});
|
|
273308
273579
|
|
|
@@ -274165,9 +274436,11 @@ function extractRunTargetSourceName(query) {
|
|
|
274165
274436
|
}
|
|
274166
274437
|
function buildSourceAliasMap(query) {
|
|
274167
274438
|
const aliasOf = new Map;
|
|
274168
|
-
const
|
|
274439
|
+
const text = stripMalloyCommentsAndLiterals(query);
|
|
274440
|
+
const ident = String.raw`(?:\x60([^\x60]+)\x60|([\p{L}\p{N}_]+))`;
|
|
274441
|
+
const declRe = new RegExp(String.raw`source\s*:\s*${ident}(?:\s*\([^)]*\))?\s+is\s*\(?\s*${ident}`, "gu");
|
|
274169
274442
|
let match;
|
|
274170
|
-
while ((match = declRe.exec(
|
|
274443
|
+
while ((match = declRe.exec(text)) !== null) {
|
|
274171
274444
|
aliasOf.set(match[1] ?? match[2], match[3] ?? match[4]);
|
|
274172
274445
|
}
|
|
274173
274446
|
return aliasOf;
|
|
@@ -274218,8 +274491,9 @@ function stripMalloyCommentsAndLiterals(text) {
|
|
|
274218
274491
|
}
|
|
274219
274492
|
return out.join("");
|
|
274220
274493
|
}
|
|
274221
|
-
function buildDerivationBaseMap(
|
|
274494
|
+
function buildDerivationBaseMap(query) {
|
|
274222
274495
|
const basesOf = new Map;
|
|
274496
|
+
const text = stripMalloyCommentsAndLiterals(query);
|
|
274223
274497
|
const ident = String.raw`(?:\x60([^\x60]+)\x60|([\p{L}\p{N}_]+))`;
|
|
274224
274498
|
const declRe = new RegExp(String.raw`(?:source|query)\s*:\s*${ident}(?:\s*\([^)]*\))?\s+is\s*\(?\s*${ident}`, "gu");
|
|
274225
274499
|
let match;
|
|
@@ -274237,7 +274511,7 @@ function buildDerivationBaseMap(text) {
|
|
|
274237
274511
|
import {
|
|
274238
274512
|
expressionIsAggregate,
|
|
274239
274513
|
expressionIsAnalytic,
|
|
274240
|
-
isJoined as
|
|
274514
|
+
isJoined as isJoined3
|
|
274241
274515
|
} from "@malloydata/malloy";
|
|
274242
274516
|
function isAccessRestricted2(f) {
|
|
274243
274517
|
return f.accessModifier != null && f.accessModifier !== "public";
|
|
@@ -274249,13 +274523,13 @@ function isView(f) {
|
|
|
274249
274523
|
return f.type === "turtle";
|
|
274250
274524
|
}
|
|
274251
274525
|
function isFanoutJoin(f) {
|
|
274252
|
-
return
|
|
274526
|
+
return isJoined3(f) && f.join !== "one";
|
|
274253
274527
|
}
|
|
274254
274528
|
function isMeasure(f) {
|
|
274255
274529
|
return expressionIsAggregate(f.expressionType) && !expressionIsAnalytic(f.expressionType);
|
|
274256
274530
|
}
|
|
274257
274531
|
function isGroupableDimension(f) {
|
|
274258
|
-
if (
|
|
274532
|
+
if (isJoined3(f) || isView(f))
|
|
274259
274533
|
return false;
|
|
274260
274534
|
if (f.expressionType === undefined)
|
|
274261
274535
|
return true;
|
|
@@ -274275,7 +274549,7 @@ function validateGrainDimension(sourceName, measure, source, grainDimension) {
|
|
|
274275
274549
|
if (!field) {
|
|
274276
274550
|
return violation("unknown_grain_dimension", `Measure \`${measure}\` declares \`#@ preaggregate\` at a grain naming \`${grainDimension}\`, but \`${head}\` is not a field of \`${sourceName}\`. Use a dimension that exists on the source; a rollup can only group by the source's own dimensions.`);
|
|
274277
274551
|
}
|
|
274278
|
-
if (
|
|
274552
|
+
if (isJoined3(field)) {
|
|
274279
274553
|
return violation("grain_path_through_join", `Measure \`${measure}\` declares a grain of \`${grainDimension}\`, which reaches through the join \`${head}\`. Pre-aggregating across a join is not supported in this version; copy the value into a dimension on \`${sourceName}\` and group by that instead.`);
|
|
274280
274554
|
}
|
|
274281
274555
|
if (parts.length > 1) {
|
|
@@ -274324,7 +274598,7 @@ function validateSourcePreaggregation(sourceName, source) {
|
|
|
274324
274598
|
if (!declaration.declared)
|
|
274325
274599
|
continue;
|
|
274326
274600
|
declaredFields.push(field);
|
|
274327
|
-
if (
|
|
274601
|
+
if (isJoined3(field)) {
|
|
274328
274602
|
violations.push({
|
|
274329
274603
|
code: "misplaced_on_join",
|
|
274330
274604
|
sourceName,
|
|
@@ -274467,7 +274741,7 @@ var init_preaggregation_validation = __esm(() => {
|
|
|
274467
274741
|
|
|
274468
274742
|
// src/service/source_extraction.ts
|
|
274469
274743
|
import {
|
|
274470
|
-
isJoined as
|
|
274744
|
+
isJoined as isJoined4,
|
|
274471
274745
|
isSourceDef as isSourceDef6,
|
|
274472
274746
|
modelDefToModelInfo
|
|
274473
274747
|
} from "@malloydata/malloy";
|
|
@@ -274521,7 +274795,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274521
274795
|
considerNoteOwner(authorizeNoteDeclaredBy, note, struct);
|
|
274522
274796
|
}
|
|
274523
274797
|
}
|
|
274524
|
-
nearMissAuthorize.push(...
|
|
274798
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
|
|
274525
274799
|
...ownLevelNotes(struct.annotations),
|
|
274526
274800
|
...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
|
|
274527
274801
|
].map((note) => note.text)));
|
|
@@ -274539,23 +274813,29 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274539
274813
|
considerNoteOwner(authorizeNoteDeclaredBy, note, entry);
|
|
274540
274814
|
}
|
|
274541
274815
|
}
|
|
274542
|
-
nearMissAuthorize.push(...
|
|
274816
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes(ownLevelNotes(entry.annotations).map((note) => note.text)));
|
|
274543
274817
|
}
|
|
274544
274818
|
for (const struct of derivedStructsReachable(sweptStructs, modelDef)) {
|
|
274545
|
-
nearMissAuthorize.push(...
|
|
274819
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
|
|
274546
274820
|
...ownLevelNotes(struct.annotations),
|
|
274547
274821
|
...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
|
|
274548
274822
|
].map((note) => note.text)));
|
|
274549
274823
|
}
|
|
274550
274824
|
{
|
|
274551
274825
|
const folded = modelAnnotations(modelDef);
|
|
274552
|
-
nearMissAuthorize.push(...
|
|
274826
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([...folded.notes ?? [], ...folded.blockNotes ?? []].map((note) => note.text)));
|
|
274553
274827
|
}
|
|
274554
274828
|
assertNoAuthorizeNearMisses(nearMissAuthorize);
|
|
274555
274829
|
const authorizeOwnNotes = new Map;
|
|
274556
274830
|
const attributedAuthorizeOwnNotes = new Map;
|
|
274557
|
-
|
|
274558
|
-
|
|
274831
|
+
{
|
|
274832
|
+
const fileNoteTexts = ownLevelNoteTexts(modelAnnotations(modelDef));
|
|
274833
|
+
if (containsAuthorizeAnnotationTag(fileNoteTexts)) {
|
|
274834
|
+
misplacedAuthorize.push({
|
|
274835
|
+
kind: "file",
|
|
274836
|
+
route: fileNoteTexts.map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274837
|
+
});
|
|
274838
|
+
}
|
|
274559
274839
|
}
|
|
274560
274840
|
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef6(obj)).map((sourceObj) => {
|
|
274561
274841
|
const struct = sourceObj;
|
|
@@ -274594,17 +274874,32 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274594
274874
|
}
|
|
274595
274875
|
}
|
|
274596
274876
|
const ownNotes = ownLevelNoteTexts(struct.annotations);
|
|
274597
|
-
const ownGates = collectAuthorizeExprs(ownNotes);
|
|
274598
274877
|
const ownAuthorizeNotes = ownLevelNotes(struct.annotations).filter((note) => containsAuthorizeAnnotationTag([note.text]));
|
|
274599
|
-
|
|
274600
|
-
|
|
274601
|
-
const
|
|
274602
|
-
|
|
274603
|
-
|
|
274604
|
-
|
|
274605
|
-
|
|
274606
|
-
|
|
274607
|
-
|
|
274878
|
+
const ownNotesByRoute = new Map;
|
|
274879
|
+
const attributedOwnNotesByRoute = new Map;
|
|
274880
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
274881
|
+
const notesForRoute = ownAuthorizeNotes.filter((note) => authorizeAnnotationRoute(note.text) === route);
|
|
274882
|
+
ownNotesByRoute.set(route, notesForRoute);
|
|
274883
|
+
attributedOwnNotesByRoute.set(route, notesForRoute.filter((note) => authorizeNoteDeclaredBy.get(note) === struct));
|
|
274884
|
+
}
|
|
274885
|
+
authorizeOwnNotes.set(sourceName, ownNotesByRoute);
|
|
274886
|
+
attributedAuthorizeOwnNotes.set(sourceName, attributedOwnNotesByRoute);
|
|
274887
|
+
const groupsForSource = [];
|
|
274888
|
+
const routeGroupsByRoute = new Map;
|
|
274889
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
274890
|
+
const ownExprsForRoute = collectAuthorizeExprsForRoute(ownNotes, route);
|
|
274891
|
+
const groupsForRoute = ownExprsForRoute.length > 0 ? [ownExprsForRoute] : effectiveAncestorGateExprs(struct, modelDef, route);
|
|
274892
|
+
for (const exprs of groupsForRoute) {
|
|
274893
|
+
groupsForSource.push({ route, exprs });
|
|
274894
|
+
}
|
|
274895
|
+
routeGroupsByRoute.set(route, groupsForRoute);
|
|
274896
|
+
}
|
|
274897
|
+
if (groupsForSource.length > 0) {
|
|
274898
|
+
authorizeMap.set(sourceName, groupsForSource);
|
|
274899
|
+
}
|
|
274900
|
+
const routeGroupsToFlatWire = (groups) => groups && groups.length > 0 ? groups.flat() : undefined;
|
|
274901
|
+
const authorize = routeGroupsToFlatWire(routeGroupsByRoute.get(AUTHORIZE_ROUTE));
|
|
274902
|
+
const accessFilter = routeGroupsToFlatWire(routeGroupsByRoute.get(ACCESS_FILTER_ROUTE));
|
|
274608
274903
|
const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
|
|
274609
274904
|
name: turtle.as || turtle.name,
|
|
274610
274905
|
annotations: annotationTexts(turtle.annotations)
|
|
@@ -274617,7 +274912,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274617
274912
|
continue;
|
|
274618
274913
|
}
|
|
274619
274914
|
const fieldName2 = field.as || field.name;
|
|
274620
|
-
if (
|
|
274915
|
+
if (isJoined4(field) && isSourceDef6(field)) {
|
|
274621
274916
|
const joinedStruct = field;
|
|
274622
274917
|
if (joinFieldNamesUnresolvableDeclaration(joinedStruct, modelDef)) {
|
|
274623
274918
|
continue;
|
|
@@ -274625,14 +274920,16 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274625
274920
|
misplacedAuthorize.push({
|
|
274626
274921
|
kind: "field",
|
|
274627
274922
|
name: sourceName,
|
|
274628
|
-
fieldName: fieldName2
|
|
274923
|
+
fieldName: fieldName2,
|
|
274924
|
+
route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274629
274925
|
});
|
|
274630
274926
|
continue;
|
|
274631
274927
|
}
|
|
274632
274928
|
misplacedAuthorize.push({
|
|
274633
274929
|
kind: "field",
|
|
274634
274930
|
name: sourceName,
|
|
274635
|
-
fieldName: fieldName2
|
|
274931
|
+
fieldName: fieldName2,
|
|
274932
|
+
route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274636
274933
|
});
|
|
274637
274934
|
}
|
|
274638
274935
|
return {
|
|
@@ -274641,7 +274938,8 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274641
274938
|
views,
|
|
274642
274939
|
filters,
|
|
274643
274940
|
givens,
|
|
274644
|
-
authorize
|
|
274941
|
+
authorize,
|
|
274942
|
+
accessFilter
|
|
274645
274943
|
};
|
|
274646
274944
|
});
|
|
274647
274945
|
return {
|
|
@@ -274658,7 +274956,8 @@ function extractQueriesFromModelDef(modelDef) {
|
|
|
274658
274956
|
const namedQueries = Object.values(modelDef.contents).filter(isNamedQuery);
|
|
274659
274957
|
const misplacedAuthorize = namedQueries.filter((queryObj) => containsAuthorizeAnnotationTag(ownLevelNoteTexts(queryObj.annotations))).map((queryObj) => ({
|
|
274660
274958
|
kind: "query",
|
|
274661
|
-
name: queryObj.as || queryObj.name
|
|
274959
|
+
name: queryObj.as || queryObj.name,
|
|
274960
|
+
route: ownLevelNoteTexts(queryObj.annotations).map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274662
274961
|
}));
|
|
274663
274962
|
const queries = namedQueries.map((queryObj) => ({
|
|
274664
274963
|
name: queryObj.as || queryObj.name,
|
|
@@ -274670,10 +274969,113 @@ function extractQueriesFromModelDef(modelDef) {
|
|
|
274670
274969
|
var init_source_extraction = __esm(() => {
|
|
274671
274970
|
init_annotations();
|
|
274672
274971
|
init_authorize();
|
|
274972
|
+
init_authorize_routes();
|
|
274673
274973
|
init_filter();
|
|
274674
274974
|
init_gate_registry_walk();
|
|
274675
274975
|
});
|
|
274676
274976
|
|
|
274977
|
+
// src/service/authorize_lock.ts
|
|
274978
|
+
function decideLock(conditionExpr, givenNameOf, givens) {
|
|
274979
|
+
const ctx = { givenNameOf, givens, unresolvable: false };
|
|
274980
|
+
try {
|
|
274981
|
+
if (admits(conditionExpr, ctx, 0))
|
|
274982
|
+
return "admit";
|
|
274983
|
+
} catch {
|
|
274984
|
+
ctx.unresolvable = true;
|
|
274985
|
+
}
|
|
274986
|
+
return ctx.unresolvable ? "unresolvable" : "deny";
|
|
274987
|
+
}
|
|
274988
|
+
function admits(node, ctx, depth) {
|
|
274989
|
+
if (depth > 64 || node === null || typeof node !== "object") {
|
|
274990
|
+
ctx.unresolvable = true;
|
|
274991
|
+
return false;
|
|
274992
|
+
}
|
|
274993
|
+
const n = node;
|
|
274994
|
+
switch (n.node) {
|
|
274995
|
+
case "()":
|
|
274996
|
+
return admits(n.e, ctx, depth + 1);
|
|
274997
|
+
case "true":
|
|
274998
|
+
return true;
|
|
274999
|
+
case "false":
|
|
275000
|
+
return false;
|
|
275001
|
+
case "and": {
|
|
275002
|
+
const left = admits(n.kids?.left, ctx, depth + 1);
|
|
275003
|
+
const right = admits(n.kids?.right, ctx, depth + 1);
|
|
275004
|
+
return left && right;
|
|
275005
|
+
}
|
|
275006
|
+
case "=": {
|
|
275007
|
+
const { left, right } = n.kids ?? {};
|
|
275008
|
+
const pair = givenAndLiteral(left, right) ?? givenAndLiteral(right, left);
|
|
275009
|
+
if (!pair) {
|
|
275010
|
+
ctx.unresolvable = true;
|
|
275011
|
+
return false;
|
|
275012
|
+
}
|
|
275013
|
+
const value = resolve4(pair.given, ctx);
|
|
275014
|
+
if (typeof value !== "string") {
|
|
275015
|
+
ctx.unresolvable = true;
|
|
275016
|
+
return false;
|
|
275017
|
+
}
|
|
275018
|
+
return value === pair.literal;
|
|
275019
|
+
}
|
|
275020
|
+
case "inGiven": {
|
|
275021
|
+
if (n.not === true)
|
|
275022
|
+
return false;
|
|
275023
|
+
const given = asGiven(n.givenRef);
|
|
275024
|
+
const literal = asStringLiteral(n.e);
|
|
275025
|
+
if (!given || literal === undefined) {
|
|
275026
|
+
ctx.unresolvable = true;
|
|
275027
|
+
return false;
|
|
275028
|
+
}
|
|
275029
|
+
const value = resolve4(given, ctx);
|
|
275030
|
+
if (!Array.isArray(value)) {
|
|
275031
|
+
ctx.unresolvable = true;
|
|
275032
|
+
return false;
|
|
275033
|
+
}
|
|
275034
|
+
return value.some((item) => item === literal);
|
|
275035
|
+
}
|
|
275036
|
+
default:
|
|
275037
|
+
ctx.unresolvable = true;
|
|
275038
|
+
return false;
|
|
275039
|
+
}
|
|
275040
|
+
}
|
|
275041
|
+
function givenAndLiteral(a, b) {
|
|
275042
|
+
const given = asGiven(a);
|
|
275043
|
+
const literal = asStringLiteral(b);
|
|
275044
|
+
if (!given || literal === undefined)
|
|
275045
|
+
return;
|
|
275046
|
+
return { given, literal };
|
|
275047
|
+
}
|
|
275048
|
+
function asGiven(node) {
|
|
275049
|
+
if (node === null || typeof node !== "object")
|
|
275050
|
+
return;
|
|
275051
|
+
const n = node;
|
|
275052
|
+
return n.node === "given" ? n : undefined;
|
|
275053
|
+
}
|
|
275054
|
+
function asStringLiteral(node) {
|
|
275055
|
+
if (node === null || typeof node !== "object")
|
|
275056
|
+
return;
|
|
275057
|
+
const n = node;
|
|
275058
|
+
if (n.node !== "stringLiteral")
|
|
275059
|
+
return;
|
|
275060
|
+
return typeof n.literal === "string" ? n.literal : undefined;
|
|
275061
|
+
}
|
|
275062
|
+
function resolve4(given, ctx) {
|
|
275063
|
+
if (typeof given.id !== "string") {
|
|
275064
|
+
ctx.unresolvable = true;
|
|
275065
|
+
return;
|
|
275066
|
+
}
|
|
275067
|
+
const name = ctx.givenNameOf(given.id);
|
|
275068
|
+
if (name === undefined) {
|
|
275069
|
+
ctx.unresolvable = true;
|
|
275070
|
+
return;
|
|
275071
|
+
}
|
|
275072
|
+
if (!Object.prototype.hasOwnProperty.call(ctx.givens, name)) {
|
|
275073
|
+
ctx.unresolvable = true;
|
|
275074
|
+
return;
|
|
275075
|
+
}
|
|
275076
|
+
return ctx.givens[name];
|
|
275077
|
+
}
|
|
275078
|
+
|
|
274677
275079
|
// src/service/model.ts
|
|
274678
275080
|
import {
|
|
274679
275081
|
API,
|
|
@@ -274695,6 +275097,22 @@ import { readFileSync as readFileSync2 } from "fs";
|
|
|
274695
275097
|
import { createRequire as createRequire2 } from "module";
|
|
274696
275098
|
import * as path10 from "path";
|
|
274697
275099
|
import { fileURLToPath as fileURLToPath6 } from "url";
|
|
275100
|
+
function denyUnlessAdmitted(resolution, givens, label) {
|
|
275101
|
+
if (resolution.shape === "lock") {
|
|
275102
|
+
const outcome = decideLock(resolution.condition.e, (id) => resolution.givenNamesById.get(id), givens);
|
|
275103
|
+
if (outcome === "admit") {
|
|
275104
|
+
recordLockDecision("admitted");
|
|
275105
|
+
return;
|
|
275106
|
+
}
|
|
275107
|
+
recordLockDecision(outcome === "unresolvable" ? "denied_unresolvable" : "denied_by_lock");
|
|
275108
|
+
throw new AccessDeniedError(`Access denied for source "${label}".`);
|
|
275109
|
+
}
|
|
275110
|
+
recordLockDecision("denied_unresolvable");
|
|
275111
|
+
recordRowLevelGateDecision("denied_by_gate");
|
|
275112
|
+
if (resolution.cause)
|
|
275113
|
+
recordRowLevelGateRejected(resolution.cause);
|
|
275114
|
+
throw new AccessDeniedError(`Access denied for source "${label}".`);
|
|
275115
|
+
}
|
|
274698
275116
|
function quoteMalloyIdentifier2(name) {
|
|
274699
275117
|
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
274700
275118
|
}
|
|
@@ -274782,6 +275200,7 @@ var init_model = __esm(() => {
|
|
|
274782
275200
|
init_annotations();
|
|
274783
275201
|
init_build_plan();
|
|
274784
275202
|
init_authorize();
|
|
275203
|
+
init_authorize_routes();
|
|
274785
275204
|
init_dashboard();
|
|
274786
275205
|
init_gate_dimension();
|
|
274787
275206
|
init_filter();
|
|
@@ -274793,7 +275212,6 @@ var init_model = __esm(() => {
|
|
|
274793
275212
|
init_query_metadata();
|
|
274794
275213
|
init_preaggregation_validation();
|
|
274795
275214
|
init_gate_registry_walk();
|
|
274796
|
-
init_partition_annotation();
|
|
274797
275215
|
init_gate_classification();
|
|
274798
275216
|
init_source_extraction();
|
|
274799
275217
|
init_authorize_metrics();
|
|
@@ -274867,9 +275285,17 @@ var init_model = __esm(() => {
|
|
|
274867
275285
|
for (const source of this.sources ?? []) {
|
|
274868
275286
|
if (!source.name)
|
|
274869
275287
|
continue;
|
|
274870
|
-
const
|
|
274871
|
-
|
|
274872
|
-
|
|
275288
|
+
const gates = this.entryPointGatesBySource.get(source.name);
|
|
275289
|
+
const exprsForRoute = (route) => {
|
|
275290
|
+
const exprs = gates?.filter((g) => g.route === route).flatMap((g) => g.exprs);
|
|
275291
|
+
return exprs && exprs.length > 0 ? exprs : undefined;
|
|
275292
|
+
};
|
|
275293
|
+
const lockExprs = exprsForRoute(AUTHORIZE_ROUTE);
|
|
275294
|
+
if (lockExprs)
|
|
275295
|
+
source.authorize = lockExprs;
|
|
275296
|
+
const filterExprs = exprsForRoute(ACCESS_FILTER_ROUTE);
|
|
275297
|
+
if (filterExprs)
|
|
275298
|
+
source.accessFilter = filterExprs;
|
|
274873
275299
|
}
|
|
274874
275300
|
try {
|
|
274875
275301
|
this.authorizeReferencedGivenNames = this.computeAuthorizeReferencedGivenNames();
|
|
@@ -274940,6 +275366,9 @@ var init_model = __esm(() => {
|
|
|
274940
275366
|
getAuthorize(sourceName) {
|
|
274941
275367
|
return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
|
|
274942
275368
|
}
|
|
275369
|
+
getAccessFilter(sourceName) {
|
|
275370
|
+
return this.sources?.find((source) => source.name === sourceName)?.accessFilter ?? [];
|
|
275371
|
+
}
|
|
274943
275372
|
filterGivensToModelSurface(givens) {
|
|
274944
275373
|
if (!givens)
|
|
274945
275374
|
return givens;
|
|
@@ -275025,28 +275454,7 @@ var init_model = __esm(() => {
|
|
|
275025
275454
|
structs.push(...derivedStructsReachable(structs, modelDef));
|
|
275026
275455
|
return structs;
|
|
275027
275456
|
}
|
|
275028
|
-
|
|
275029
|
-
hasAnyPartitionNote() {
|
|
275030
|
-
if (this.anyPartitionNote !== undefined)
|
|
275031
|
-
return this.anyPartitionNote;
|
|
275032
|
-
this.anyPartitionNote = (() => {
|
|
275033
|
-
const modelDef = this.modelDef;
|
|
275034
|
-
if (!modelDef)
|
|
275035
|
-
return false;
|
|
275036
|
-
try {
|
|
275037
|
-
for (const struct of this.reachableStructsForNoteSweep(modelDef)) {
|
|
275038
|
-
if (containsPartitionAnnotationTag(annotationTexts(struct.annotations) ?? [])) {
|
|
275039
|
-
return true;
|
|
275040
|
-
}
|
|
275041
|
-
}
|
|
275042
|
-
return false;
|
|
275043
|
-
} catch {
|
|
275044
|
-
return true;
|
|
275045
|
-
}
|
|
275046
|
-
})();
|
|
275047
|
-
return this.anyPartitionNote;
|
|
275048
|
-
}
|
|
275049
|
-
async assertAuthorized(sourceName, _givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
|
|
275457
|
+
async assertAuthorized(sourceName, givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
|
|
275050
275458
|
if (bypassAuthorize) {
|
|
275051
275459
|
this.noteAuthorizeBypass("source", sourceName);
|
|
275052
275460
|
return;
|
|
@@ -275057,10 +275465,7 @@ var init_model = __esm(() => {
|
|
|
275057
275465
|
const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
275058
275466
|
if (resolution.shape === "row_level")
|
|
275059
275467
|
continue;
|
|
275060
|
-
|
|
275061
|
-
if (resolution.cause)
|
|
275062
|
-
recordRowLevelGateRejected(resolution.cause);
|
|
275063
|
-
throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
|
|
275468
|
+
denyUnlessAdmitted(resolution, givens, entry.label);
|
|
275064
275469
|
}
|
|
275065
275470
|
return;
|
|
275066
275471
|
}
|
|
@@ -275095,7 +275500,7 @@ var init_model = __esm(() => {
|
|
|
275095
275500
|
const onDiskGates = this.entryPointGatesBySource.get(ownSourceName);
|
|
275096
275501
|
if (onDiskGates) {
|
|
275097
275502
|
const SEP = "\x1F";
|
|
275098
|
-
const keyOf = (entry) => `${entry.label}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
|
|
275503
|
+
const keyOf = (entry) => `${entry.label}${SEP}${entry.route}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
|
|
275099
275504
|
const byKey = new Map(entryPointGates.map((entry) => [keyOf(entry), entry]));
|
|
275100
275505
|
for (const entry of onDiskGates) {
|
|
275101
275506
|
byKey.set(keyOf(entry), entry);
|
|
@@ -275115,7 +275520,7 @@ var init_model = __esm(() => {
|
|
|
275115
275520
|
return this.rowLevelFilteredRunnables.has(runnable);
|
|
275116
275521
|
}
|
|
275117
275522
|
async probeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate = false) {
|
|
275118
|
-
const { entryPointGates, modelDef
|
|
275523
|
+
const { entryPointGates, modelDef } = await this.collectAuthorizeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate);
|
|
275119
275524
|
const rowLevel = [];
|
|
275120
275525
|
for (const entry of entryPointGates) {
|
|
275121
275526
|
const resolution = modelDef ? await this.resolveGateShape(entry, modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
@@ -275129,20 +275534,7 @@ var init_model = __esm(() => {
|
|
|
275129
275534
|
});
|
|
275130
275535
|
continue;
|
|
275131
275536
|
}
|
|
275132
|
-
|
|
275133
|
-
if (resolution.cause)
|
|
275134
|
-
recordRowLevelGateRejected(resolution.cause);
|
|
275135
|
-
throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
|
|
275136
|
-
}
|
|
275137
|
-
try {
|
|
275138
|
-
rowLevel.push(...await resolvePartitionGraftEntries(struct, modelDef, graftScope, this.gateClassificationDeps()));
|
|
275139
|
-
} catch (err) {
|
|
275140
|
-
recordRowLevelGateDecision("denied_by_gate");
|
|
275141
|
-
logger.debug("Partition filter could not be resolved; denying", {
|
|
275142
|
-
modelPath: this.modelPath,
|
|
275143
|
-
error: err instanceof Error ? err.message : String(err)
|
|
275144
|
-
});
|
|
275145
|
-
throw new AccessDeniedError(`Access denied for source "${struct?.as ?? struct?.name ?? "unknown"}".`);
|
|
275537
|
+
denyUnlessAdmitted(resolution, givens, entry.label);
|
|
275146
275538
|
}
|
|
275147
275539
|
return rowLevel;
|
|
275148
275540
|
}
|
|
@@ -275156,12 +275548,12 @@ var init_model = __esm(() => {
|
|
|
275156
275548
|
if (compositeResolvedSourceDef) {
|
|
275157
275549
|
gates.push(...this.collectEntryPointGates(compositeResolvedSourceDef, modelDef, seen, true, undefined, struct ? ownLevelNotes(struct.annotations) : []));
|
|
275158
275550
|
}
|
|
275159
|
-
return gates.length > 0
|
|
275551
|
+
return gates.length > 0;
|
|
275160
275552
|
} catch {
|
|
275161
275553
|
return true;
|
|
275162
275554
|
}
|
|
275163
275555
|
}
|
|
275164
|
-
async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query) {
|
|
275556
|
+
async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query, givens, graftScope) {
|
|
275165
275557
|
if (!this.declaresAnyGate())
|
|
275166
275558
|
return;
|
|
275167
275559
|
const entryPoint = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
@@ -275170,15 +275562,51 @@ var init_model = __esm(() => {
|
|
|
275170
275562
|
if (this.entryPointGatesBySource.has(entryPoint))
|
|
275171
275563
|
return;
|
|
275172
275564
|
const proof = this.requestChainProvesUngated(entryPoint, buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query)));
|
|
275173
|
-
if (proof.proven)
|
|
275565
|
+
if (proof.proven) {
|
|
275566
|
+
for (const base of proof.gatedBases) {
|
|
275567
|
+
for (const entry of this.entryPointGatesBySource.get(base) ?? []) {
|
|
275568
|
+
const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
275569
|
+
if (resolution.shape === "row_level") {
|
|
275570
|
+
this.denyLaunderedEntryPoint(entryPoint, "row_filter_not_graftable", base);
|
|
275571
|
+
}
|
|
275572
|
+
denyUnlessAdmitted(resolution, givens, entryPoint);
|
|
275573
|
+
}
|
|
275574
|
+
}
|
|
275174
275575
|
return;
|
|
275576
|
+
}
|
|
275577
|
+
this.denyLaunderedEntryPoint(entryPoint, proof.reason, proof.at);
|
|
275578
|
+
}
|
|
275579
|
+
async assertLocksOnRequestDeclaredBases(entryPoint, query, givens, bypassAuthorize) {
|
|
275580
|
+
if (!this.declaresAnyGate())
|
|
275581
|
+
return;
|
|
275582
|
+
const basesOf = buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query));
|
|
275583
|
+
const seen = new Set;
|
|
275584
|
+
const worklist = entryPoint ? [entryPoint] : [...basesOf.keys()];
|
|
275585
|
+
for (let i = 0;i < worklist.length; i++) {
|
|
275586
|
+
const name = worklist[i];
|
|
275587
|
+
if (seen.has(name))
|
|
275588
|
+
continue;
|
|
275589
|
+
seen.add(name);
|
|
275590
|
+
if (seen.size > REQUEST_CHAIN_MAX_NAMES)
|
|
275591
|
+
return;
|
|
275592
|
+
if (this.entryPointGatesBySource.has(name)) {
|
|
275593
|
+
try {
|
|
275594
|
+
await this.assertAuthorized(name, givens, bypassAuthorize);
|
|
275595
|
+
} catch (error) {
|
|
275596
|
+
if (error instanceof AccessDeniedError) {
|
|
275597
|
+
this.assertQueryBoundaryEarly(name, undefined, undefined);
|
|
275598
|
+
}
|
|
275599
|
+
throw error;
|
|
275600
|
+
}
|
|
275601
|
+
continue;
|
|
275602
|
+
}
|
|
275603
|
+
for (const base of basesOf.get(name) ?? [])
|
|
275604
|
+
worklist.push(base);
|
|
275605
|
+
}
|
|
275606
|
+
}
|
|
275607
|
+
denyLaunderedEntryPoint(entryPoint, reason, at) {
|
|
275175
275608
|
recordRowLevelGateDecision("denied_by_gate");
|
|
275176
|
-
logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", {
|
|
275177
|
-
modelPath: this.modelPath,
|
|
275178
|
-
entryPoint,
|
|
275179
|
-
reason: proof.reason,
|
|
275180
|
-
at: proof.at
|
|
275181
|
-
});
|
|
275609
|
+
logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", { modelPath: this.modelPath, entryPoint, reason, at });
|
|
275182
275610
|
throw new AccessDeniedError(`Access denied for source "${entryPoint}".`);
|
|
275183
275611
|
}
|
|
275184
275612
|
declaresAnyGateMemo;
|
|
@@ -275188,6 +275616,7 @@ var init_model = __esm(() => {
|
|
|
275188
275616
|
}
|
|
275189
275617
|
requestChainProvesUngated(entryPoint, basesOf) {
|
|
275190
275618
|
const seen = new Set;
|
|
275619
|
+
const gatedBases = [];
|
|
275191
275620
|
const worklist = [entryPoint];
|
|
275192
275621
|
for (let i = 0;i < worklist.length; i++) {
|
|
275193
275622
|
const name = worklist[i];
|
|
@@ -275199,13 +275628,8 @@ var init_model = __esm(() => {
|
|
|
275199
275628
|
}
|
|
275200
275629
|
const modelGates = this.entryPointGatesBySource.get(name);
|
|
275201
275630
|
if (modelGates !== undefined) {
|
|
275202
|
-
if (modelGates.length > 0)
|
|
275203
|
-
|
|
275204
|
-
proven: false,
|
|
275205
|
-
reason: "reaches_gated_source",
|
|
275206
|
-
at: name
|
|
275207
|
-
};
|
|
275208
|
-
}
|
|
275631
|
+
if (modelGates.length > 0)
|
|
275632
|
+
gatedBases.push(name);
|
|
275209
275633
|
continue;
|
|
275210
275634
|
}
|
|
275211
275635
|
const bases = basesOf.get(name);
|
|
@@ -275215,7 +275639,7 @@ var init_model = __esm(() => {
|
|
|
275215
275639
|
for (const base of bases)
|
|
275216
275640
|
worklist.push(base);
|
|
275217
275641
|
}
|
|
275218
|
-
return { proven: true };
|
|
275642
|
+
return { proven: true, gatedBases };
|
|
275219
275643
|
}
|
|
275220
275644
|
async authorizeAndBindRunnable(runnable, givens, options) {
|
|
275221
275645
|
if (options?.bypassAuthorize) {
|
|
@@ -275226,7 +275650,7 @@ var init_model = __esm(() => {
|
|
|
275226
275650
|
const rowLevel = await this.probeEntryPointGates(runnable, givens, graftScope, options?.skipOwnSourceGate ?? false);
|
|
275227
275651
|
if (rowLevel.length === 0) {
|
|
275228
275652
|
if (options?.callerQueryText && !options?.skipOwnSourceGate) {
|
|
275229
|
-
await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText);
|
|
275653
|
+
await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText, givens, graftScope);
|
|
275230
275654
|
}
|
|
275231
275655
|
return runnable;
|
|
275232
275656
|
}
|
|
@@ -275291,7 +275715,7 @@ var init_model = __esm(() => {
|
|
|
275291
275715
|
}
|
|
275292
275716
|
async resolveGateShape(entry, originModelDef, graftScope) {
|
|
275293
275717
|
const result = await resolveGateShape(entry, originModelDef, graftScope, this.gateClassificationDeps());
|
|
275294
|
-
if (result.shape
|
|
275718
|
+
if (result.shape !== "rejected") {
|
|
275295
275719
|
for (const name of result.givenNames) {
|
|
275296
275720
|
this.authorizeReferencedGivenNames.add(name);
|
|
275297
275721
|
}
|
|
@@ -275432,7 +275856,11 @@ var init_model = __esm(() => {
|
|
|
275432
275856
|
return false;
|
|
275433
275857
|
}
|
|
275434
275858
|
async assertAuthorizedForText(text, givens) {
|
|
275435
|
-
|
|
275859
|
+
const target = extractRunTargetSourceName(text);
|
|
275860
|
+
await this.assertAuthorized(target, givens);
|
|
275861
|
+
if (!hasCallerAuthorizeAnnotation(text)) {
|
|
275862
|
+
await this.assertLocksOnRequestDeclaredBases(target ?? "", text, givens);
|
|
275863
|
+
}
|
|
275436
275864
|
}
|
|
275437
275865
|
async assertAuthorizedForRunnable(runnable, givens) {
|
|
275438
275866
|
await this.assertAuthorizedForAllSources(runnable, givens, false, {
|
|
@@ -275490,7 +275918,7 @@ var init_model = __esm(() => {
|
|
|
275490
275918
|
filterMap = sourceResult.filterMap;
|
|
275491
275919
|
const queryResult = Model.getQueries(modelDef);
|
|
275492
275920
|
queries = queryResult.queries;
|
|
275493
|
-
|
|
275921
|
+
assertNoRetiredRouteMarkers(collectRetiredRouteMarkers(modelDef));
|
|
275494
275922
|
assertNoMisplacedAuthorizeAnnotations([
|
|
275495
275923
|
...sourceResult.misplacedAuthorize,
|
|
275496
275924
|
...queryResult.misplacedAuthorize
|
|
@@ -275498,20 +275926,20 @@ var init_model = __esm(() => {
|
|
|
275498
275926
|
const legacyStringGates = findLegacyStringGates(sourceResult.authorizeOwnNotes);
|
|
275499
275927
|
legacyStringGates.forEach(() => recordRowLevelGateRejected("legacy_string_gate"));
|
|
275500
275928
|
assertNoLegacyStringGate(legacyStringGates);
|
|
275501
|
-
|
|
275929
|
+
assertAuthorizeGrammarValid(modelDef, sourceResult.authorizeMap, sourceResult.authorizeOwnNotes, computeGivenDeclaredTypes(givens), (_sourceName, route) => recordAuthorizeAdmitAllGate(route), sourceResult.attributedAuthorizeOwnNotes);
|
|
275502
275930
|
const compiledModelDef = modelDef;
|
|
275503
275931
|
await validateAuthorizeProbes(modelMaterializer, {
|
|
275504
275932
|
authorizeMap: sourceResult.authorizeMap,
|
|
275505
275933
|
authorizeOwnNotes: sourceResult.attributedAuthorizeOwnNotes,
|
|
275506
275934
|
onRowLevelGateRejected: recordRowLevelGateRejected,
|
|
275507
|
-
onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(
|
|
275508
|
-
onOwnRowLevelConditionCompiled: (sourceName, condition) => {
|
|
275935
|
+
onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(access_filter) gate not expressible at this entry point; every query against it will be denied", { packageName, modelPath, sourceName, detail }),
|
|
275936
|
+
onOwnRowLevelConditionCompiled: (sourceName, condition, route) => {
|
|
275509
275937
|
const struct = compiledModelDef.contents[sourceName];
|
|
275510
275938
|
if (!struct || !isSourceDef7(struct))
|
|
275511
275939
|
return;
|
|
275512
|
-
validateSourceLineGateGivenUsage(sourceName, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
|
|
275940
|
+
validateSourceLineGateGivenUsage(sourceName, route, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
|
|
275513
275941
|
recordRowLevelGateRejected(cause);
|
|
275514
|
-
logger.warn(
|
|
275942
|
+
logger.warn(`#(${route}) gate warning`, {
|
|
275515
275943
|
packageName,
|
|
275516
275944
|
modelPath,
|
|
275517
275945
|
sourceName,
|
|
@@ -275647,7 +276075,12 @@ var init_model = __esm(() => {
|
|
|
275647
276075
|
getDashboardModelFacts() {
|
|
275648
276076
|
if (!this.modelDef)
|
|
275649
276077
|
return;
|
|
275650
|
-
return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [
|
|
276078
|
+
return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [
|
|
276079
|
+
[
|
|
276080
|
+
source.name,
|
|
276081
|
+
gateGivenSource(this.sources ?? [], source.name) ?? []
|
|
276082
|
+
]
|
|
276083
|
+
] : [])));
|
|
275651
276084
|
}
|
|
275652
276085
|
hasEmptyDiscoverySurface() {
|
|
275653
276086
|
if (!this.discoveryCurationEnabled)
|
|
@@ -275682,17 +276115,15 @@ var init_model = __esm(() => {
|
|
|
275682
276115
|
return this.admittedByPackage(name, this.queryBoundary.packageCuratedSources);
|
|
275683
276116
|
}
|
|
275684
276117
|
isCuratedQuery(name) {
|
|
275685
|
-
if ((this.getQueries() ?? []).some((q) => q.name === name))
|
|
276118
|
+
if (!this.isNotebook() && (this.getQueries() ?? []).some((q) => q.name === name))
|
|
275686
276119
|
return true;
|
|
275687
276120
|
return this.admittedByPackage(name, this.queryBoundary.packageCuratedQueries);
|
|
275688
276121
|
}
|
|
275689
276122
|
assertQueryBoundaryEarly(sourceName, queryName, query) {
|
|
275690
|
-
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
275691
|
-
return "cleared";
|
|
275692
276123
|
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
275693
276124
|
if (mode === "all" || !exploresDeclared)
|
|
275694
276125
|
return "cleared";
|
|
275695
|
-
if (!isQueryEntryPoint) {
|
|
276126
|
+
if (!isQueryEntryPoint && !this.isNotebook()) {
|
|
275696
276127
|
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
275697
276128
|
}
|
|
275698
276129
|
if (queryName && !query) {
|
|
@@ -275710,18 +276141,16 @@ var init_model = __esm(() => {
|
|
|
275710
276141
|
if (query) {
|
|
275711
276142
|
const target = extractRunTargetSourceName(query);
|
|
275712
276143
|
if (target && !this.isCuratedSource(target) && !this.derivesFromCurated(target, query) && this.sources?.some((s) => s.name === target)) {
|
|
275713
|
-
throw new NotQueryableError(
|
|
276144
|
+
throw new NotQueryableError("Query target is not queryable.");
|
|
275714
276145
|
}
|
|
275715
276146
|
}
|
|
275716
276147
|
return "deferred";
|
|
275717
276148
|
}
|
|
275718
276149
|
assertQueryBoundaryCompiled(compiledSource, query) {
|
|
275719
|
-
if (this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX))
|
|
275720
|
-
return;
|
|
275721
276150
|
const { mode, exploresDeclared, isQueryEntryPoint } = this.queryBoundary;
|
|
275722
276151
|
if (mode === "all" || !exploresDeclared)
|
|
275723
276152
|
return;
|
|
275724
|
-
if (!isQueryEntryPoint) {
|
|
276153
|
+
if (!isQueryEntryPoint && !this.isNotebook()) {
|
|
275725
276154
|
throw new NotQueryableError(`No queryable model "${this.modelPath}".`);
|
|
275726
276155
|
}
|
|
275727
276156
|
if (compiledSource) {
|
|
@@ -275738,7 +276167,12 @@ var init_model = __esm(() => {
|
|
|
275738
276167
|
return;
|
|
275739
276168
|
this.assertQueryBoundaryCompiled(await this.resolveAuthorizeSourceFromRunnable(runnable), query);
|
|
275740
276169
|
}
|
|
276170
|
+
isNotebook() {
|
|
276171
|
+
return this.modelPath.endsWith(NOTEBOOK_FILE_SUFFIX);
|
|
276172
|
+
}
|
|
275741
276173
|
ownCuratedSourceNames() {
|
|
276174
|
+
if (this.isNotebook())
|
|
276175
|
+
return new Set;
|
|
275742
276176
|
return new Set((this.getSources() ?? []).map((s) => s.name).filter((n) => n !== undefined));
|
|
275743
276177
|
}
|
|
275744
276178
|
derivesFromCurated(name, query) {
|
|
@@ -276127,6 +276561,9 @@ var init_model = __esm(() => {
|
|
|
276127
276561
|
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || (surfaceName && !this.sources?.some((s) => s.name === surfaceName) ? this.queries?.find((q) => q.name === surfaceName)?.sourceName : undefined) || surfaceName;
|
|
276128
276562
|
if (earlySource) {
|
|
276129
276563
|
await this.assertAuthorized(earlySource, givens ?? {}, bypassAuthorize);
|
|
276564
|
+
if (query && !hasCallerAuthorizeAnnotation(query) && !this.entryPointGatesBySource.has(earlySource)) {
|
|
276565
|
+
await this.assertLocksOnRequestDeclaredBases(earlySource, query, givens ?? {}, bypassAuthorize);
|
|
276566
|
+
}
|
|
276130
276567
|
}
|
|
276131
276568
|
try {
|
|
276132
276569
|
for (const [field, callerText] of [
|
|
@@ -276175,7 +276612,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
|
|
|
276175
276612
|
liveRunnable = runnable;
|
|
276176
276613
|
const storageRoutingPossible = getPersistStorageMode() === "on" && this.serveBindings.length > 0 && !!this.serveDestinationConfig;
|
|
276177
276614
|
const preaggServeMaterializer = this.preaggregateServeMaterializer;
|
|
276178
|
-
const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize &&
|
|
276615
|
+
const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize && this.hasAnyAuthorizeNote() && await this.queryEntryPointHasRowLevelGate(runnable);
|
|
276179
276616
|
if (routingBlockedByRowLevelGate) {
|
|
276180
276617
|
recordStorageServeRouting("blocked_by_row_level_gate");
|
|
276181
276618
|
}
|
|
@@ -277659,12 +278096,12 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
277659
278096
|
ignores = [];
|
|
277660
278097
|
}
|
|
277661
278098
|
if (!callback) {
|
|
277662
|
-
return new Promise(function(
|
|
278099
|
+
return new Promise(function(resolve5, reject2) {
|
|
277663
278100
|
readdir3(path11, ignores || [], function(err, data) {
|
|
277664
278101
|
if (err) {
|
|
277665
278102
|
reject2(err);
|
|
277666
278103
|
} else {
|
|
277667
|
-
|
|
278104
|
+
resolve5(data);
|
|
277668
278105
|
}
|
|
277669
278106
|
});
|
|
277670
278107
|
});
|
|
@@ -284918,7 +285355,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
284918
285355
|
target.fragment = relative5.fragment;
|
|
284919
285356
|
return target;
|
|
284920
285357
|
}
|
|
284921
|
-
function
|
|
285358
|
+
function resolve6(baseURI, relativeURI, options) {
|
|
284922
285359
|
var schemelessOptions = assign({ scheme: "null" }, options);
|
|
284923
285360
|
return serialize(resolveComponents(parse(baseURI, schemelessOptions), parse(relativeURI, schemelessOptions), schemelessOptions, true), schemelessOptions);
|
|
284924
285361
|
}
|
|
@@ -285186,7 +285623,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
285186
285623
|
exports2.removeDotSegments = removeDotSegments;
|
|
285187
285624
|
exports2.serialize = serialize;
|
|
285188
285625
|
exports2.resolveComponents = resolveComponents;
|
|
285189
|
-
exports2.resolve =
|
|
285626
|
+
exports2.resolve = resolve6;
|
|
285190
285627
|
exports2.normalize = normalize2;
|
|
285191
285628
|
exports2.equal = equal;
|
|
285192
285629
|
exports2.escapeComponent = escapeComponent;
|
|
@@ -285552,20 +285989,20 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
285552
285989
|
var util5 = require_util12();
|
|
285553
285990
|
var SchemaObject = require_schema_obj();
|
|
285554
285991
|
var traverse = require_json_schema_traverse();
|
|
285555
|
-
module.exports =
|
|
285556
|
-
|
|
285557
|
-
|
|
285558
|
-
|
|
285559
|
-
|
|
285560
|
-
|
|
285561
|
-
|
|
285562
|
-
function
|
|
285992
|
+
module.exports = resolve6;
|
|
285993
|
+
resolve6.normalizeId = normalizeId;
|
|
285994
|
+
resolve6.fullPath = getFullPath;
|
|
285995
|
+
resolve6.url = resolveUrl;
|
|
285996
|
+
resolve6.ids = resolveIds;
|
|
285997
|
+
resolve6.inlineRef = inlineRef;
|
|
285998
|
+
resolve6.schema = resolveSchema;
|
|
285999
|
+
function resolve6(compile, root, ref) {
|
|
285563
286000
|
var refVal = this._refs[ref];
|
|
285564
286001
|
if (typeof refVal == "string") {
|
|
285565
286002
|
if (this._refs[refVal])
|
|
285566
286003
|
refVal = this._refs[refVal];
|
|
285567
286004
|
else
|
|
285568
|
-
return
|
|
286005
|
+
return resolve6.call(this, compile, root, refVal);
|
|
285569
286006
|
}
|
|
285570
286007
|
refVal = refVal || this._schemas[ref];
|
|
285571
286008
|
if (refVal instanceof SchemaObject) {
|
|
@@ -285788,7 +286225,7 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
285788
286225
|
|
|
285789
286226
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/error_classes.js
|
|
285790
286227
|
var require_error_classes = __commonJS((exports, module) => {
|
|
285791
|
-
var
|
|
286228
|
+
var resolve6 = require_resolve2();
|
|
285792
286229
|
module.exports = {
|
|
285793
286230
|
Validation: errorSubclass(ValidationError),
|
|
285794
286231
|
MissingRef: errorSubclass(MissingRefError)
|
|
@@ -285803,8 +286240,8 @@ var require_error_classes = __commonJS((exports, module) => {
|
|
|
285803
286240
|
};
|
|
285804
286241
|
function MissingRefError(baseId, ref, message) {
|
|
285805
286242
|
this.message = message || MissingRefError.message(baseId, ref);
|
|
285806
|
-
this.missingRef =
|
|
285807
|
-
this.missingSchema =
|
|
286243
|
+
this.missingRef = resolve6.url(baseId, ref);
|
|
286244
|
+
this.missingSchema = resolve6.normalizeId(resolve6.fullPath(this.missingRef));
|
|
285808
286245
|
}
|
|
285809
286246
|
function errorSubclass(Subclass) {
|
|
285810
286247
|
Subclass.prototype = Object.create(Error.prototype);
|
|
@@ -286346,7 +286783,7 @@ var require_validate4 = __commonJS((exports, module) => {
|
|
|
286346
286783
|
|
|
286347
286784
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/index.js
|
|
286348
286785
|
var require_compile2 = __commonJS((exports, module) => {
|
|
286349
|
-
var
|
|
286786
|
+
var resolve6 = require_resolve2();
|
|
286350
286787
|
var util5 = require_util12();
|
|
286351
286788
|
var errorClasses = require_error_classes();
|
|
286352
286789
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -286406,7 +286843,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286406
286843
|
RULES,
|
|
286407
286844
|
validate: validateGenerator,
|
|
286408
286845
|
util: util5,
|
|
286409
|
-
resolve:
|
|
286846
|
+
resolve: resolve6,
|
|
286410
286847
|
resolveRef,
|
|
286411
286848
|
usePattern,
|
|
286412
286849
|
useDefault,
|
|
@@ -286445,7 +286882,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286445
286882
|
return validate;
|
|
286446
286883
|
}
|
|
286447
286884
|
function resolveRef(baseId2, ref, isRoot) {
|
|
286448
|
-
ref =
|
|
286885
|
+
ref = resolve6.url(baseId2, ref);
|
|
286449
286886
|
var refIndex = refs[ref];
|
|
286450
286887
|
var _refVal, refCode;
|
|
286451
286888
|
if (refIndex !== undefined) {
|
|
@@ -286462,11 +286899,11 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286462
286899
|
}
|
|
286463
286900
|
}
|
|
286464
286901
|
refCode = addLocalRef(ref);
|
|
286465
|
-
var v2 =
|
|
286902
|
+
var v2 = resolve6.call(self2, localCompile, root, ref);
|
|
286466
286903
|
if (v2 === undefined) {
|
|
286467
286904
|
var localSchema = localRefs && localRefs[ref];
|
|
286468
286905
|
if (localSchema) {
|
|
286469
|
-
v2 =
|
|
286906
|
+
v2 = resolve6.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
|
|
286470
286907
|
}
|
|
286471
286908
|
}
|
|
286472
286909
|
if (v2 === undefined) {
|
|
@@ -289983,7 +290420,7 @@ var require_data2 = __commonJS((exports, module) => {
|
|
|
289983
290420
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/ajv.js
|
|
289984
290421
|
var require_ajv = __commonJS((exports, module) => {
|
|
289985
290422
|
var compileSchema = require_compile2();
|
|
289986
|
-
var
|
|
290423
|
+
var resolve6 = require_resolve2();
|
|
289987
290424
|
var Cache = require_cache();
|
|
289988
290425
|
var SchemaObject = require_schema_obj();
|
|
289989
290426
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -290075,7 +290512,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290075
290512
|
var id = this._getId(schema);
|
|
290076
290513
|
if (id !== undefined && typeof id != "string")
|
|
290077
290514
|
throw new Error("schema id must be string");
|
|
290078
|
-
key =
|
|
290515
|
+
key = resolve6.normalizeId(key || id);
|
|
290079
290516
|
checkUnique(this, key);
|
|
290080
290517
|
this._schemas[key] = this._addSchema(schema, _skipValidation, _meta, true);
|
|
290081
290518
|
return this;
|
|
@@ -290121,7 +290558,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290121
290558
|
}
|
|
290122
290559
|
}
|
|
290123
290560
|
function _getSchemaFragment(self2, ref) {
|
|
290124
|
-
var res =
|
|
290561
|
+
var res = resolve6.schema.call(self2, { schema: {} }, ref);
|
|
290125
290562
|
if (res) {
|
|
290126
290563
|
var { schema, root, baseId } = res;
|
|
290127
290564
|
var v = compileSchema.call(self2, schema, root, undefined, baseId);
|
|
@@ -290137,7 +290574,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290137
290574
|
}
|
|
290138
290575
|
}
|
|
290139
290576
|
function _getSchemaObj(self2, keyRef) {
|
|
290140
|
-
keyRef =
|
|
290577
|
+
keyRef = resolve6.normalizeId(keyRef);
|
|
290141
290578
|
return self2._schemas[keyRef] || self2._refs[keyRef] || self2._fragments[keyRef];
|
|
290142
290579
|
}
|
|
290143
290580
|
function removeSchema(schemaKeyRef) {
|
|
@@ -290165,7 +290602,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290165
290602
|
this._cache.del(cacheKey);
|
|
290166
290603
|
var id = this._getId(schemaKeyRef);
|
|
290167
290604
|
if (id) {
|
|
290168
|
-
id =
|
|
290605
|
+
id = resolve6.normalizeId(id);
|
|
290169
290606
|
delete this._schemas[id];
|
|
290170
290607
|
delete this._refs[id];
|
|
290171
290608
|
}
|
|
@@ -290190,14 +290627,14 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290190
290627
|
if (cached2)
|
|
290191
290628
|
return cached2;
|
|
290192
290629
|
shouldAddSchema = shouldAddSchema || this._opts.addUsedSchema !== false;
|
|
290193
|
-
var id =
|
|
290630
|
+
var id = resolve6.normalizeId(this._getId(schema));
|
|
290194
290631
|
if (id && shouldAddSchema)
|
|
290195
290632
|
checkUnique(this, id);
|
|
290196
290633
|
var willValidate = this._opts.validateSchema !== false && !skipValidation;
|
|
290197
290634
|
var recursiveMeta;
|
|
290198
|
-
if (willValidate && !(recursiveMeta = id && id ==
|
|
290635
|
+
if (willValidate && !(recursiveMeta = id && id == resolve6.normalizeId(schema.$schema)))
|
|
290199
290636
|
this.validateSchema(schema, true);
|
|
290200
|
-
var localRefs =
|
|
290637
|
+
var localRefs = resolve6.ids.call(this, schema);
|
|
290201
290638
|
var schemaObj = new SchemaObject({
|
|
290202
290639
|
id,
|
|
290203
290640
|
schema,
|
|
@@ -299430,7 +299867,7 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
|
|
|
299430
299867
|
refined = true;
|
|
299431
299868
|
const [, sourceName] = accessDeniedMatch;
|
|
299432
299869
|
suggestions = [
|
|
299433
|
-
`Suggestion: Access to source '${sourceName}' is restricted by an
|
|
299870
|
+
`Suggestion: Access to source '${sourceName}' is restricted by an authorize gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
|
|
299434
299871
|
];
|
|
299435
299872
|
} else if (viewNotFoundMatch) {
|
|
299436
299873
|
refined = true;
|
|
@@ -300544,12 +300981,15 @@ var REASON_BY_UNAVAILABLE = {
|
|
|
300544
300981
|
"too-many-entities": "too-many-entities",
|
|
300545
300982
|
error: "provider-error"
|
|
300546
300983
|
};
|
|
300547
|
-
function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false) {
|
|
300984
|
+
function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false, droppedSources = new Set) {
|
|
300548
300985
|
const bySource = new Map;
|
|
300549
300986
|
const cardFor = (name, modelPathFallback) => {
|
|
300550
300987
|
if (!name)
|
|
300551
300988
|
return;
|
|
300552
300989
|
const key = sourceContextKey(modelPathFallback, name);
|
|
300990
|
+
if (droppedSources.has(key)) {
|
|
300991
|
+
return;
|
|
300992
|
+
}
|
|
300553
300993
|
let entry = bySource.get(key);
|
|
300554
300994
|
if (!entry) {
|
|
300555
300995
|
const ctx = sourceContext.get(key);
|
|
@@ -300564,6 +301004,7 @@ function toSourceResults(results, sourceContext, environmentName, packageName, s
|
|
|
300564
301004
|
...ctx?.oneLineSummary ? { one_line_summary: ctx.oneLineSummary } : {},
|
|
300565
301005
|
...ctx?.doc ? { docs: ctx.doc } : {},
|
|
300566
301006
|
...ctx?.givens ? { givens: ctx.givens } : {},
|
|
301007
|
+
...ctx?.accessFilter ? { accessFilter: ctx.accessFilter } : {},
|
|
300567
301008
|
...ctx?.authorize ? { authorize: ctx.authorize } : {},
|
|
300568
301009
|
...ctx?.filters ? { filter_params: ctx.filters } : {},
|
|
300569
301010
|
joins: ctx?.joins ?? []
|
|
@@ -300684,7 +301125,7 @@ function projectEntity(e, environmentName, packageName) {
|
|
|
300684
301125
|
}
|
|
300685
301126
|
function finishRanked(args) {
|
|
300686
301127
|
const windowed = windowBySource(args.rows, args.max);
|
|
300687
|
-
const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode);
|
|
301128
|
+
const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode, args.droppedSources);
|
|
300688
301129
|
return {
|
|
300689
301130
|
sources,
|
|
300690
301131
|
totalSources: windowed.totalSources,
|
|
@@ -300931,10 +301372,15 @@ function collectJoinedFields(args) {
|
|
|
300931
301372
|
});
|
|
300932
301373
|
}
|
|
300933
301374
|
}
|
|
301375
|
+
function isUnconditionalDenyAuthorize(apiSource) {
|
|
301376
|
+
const isDeny = (expr) => expr.trim().toLowerCase() === "false";
|
|
301377
|
+
return (apiSource.accessFilter ?? []).some(isDeny) || (apiSource.authorize ?? []).some(isDeny);
|
|
301378
|
+
}
|
|
300934
301379
|
async function collectEntities(pkg) {
|
|
300935
301380
|
const models = [...await pkg.listModels()].sort((a, b) => comparePaths(a.path ?? "", b.path ?? ""));
|
|
300936
301381
|
const entities = [];
|
|
300937
301382
|
const governance = new Map;
|
|
301383
|
+
const droppedSources = new Set;
|
|
300938
301384
|
const sourceTextFor = makeSourceTextReader(pkg);
|
|
300939
301385
|
let n = 0;
|
|
300940
301386
|
for (const apiModel of models) {
|
|
@@ -300948,10 +301394,18 @@ async function collectEntities(pkg) {
|
|
|
300948
301394
|
const queries = model.getQueries() ?? [];
|
|
300949
301395
|
const apiSources = model.getSources?.() ?? [];
|
|
300950
301396
|
const modelDef = model.getModelDef?.();
|
|
301397
|
+
for (const apiSource of apiSources) {
|
|
301398
|
+
if (apiSource.name && isUnconditionalDenyAuthorize(apiSource)) {
|
|
301399
|
+
droppedSources.add(sourceContextKey(modelPath, apiSource.name));
|
|
301400
|
+
}
|
|
301401
|
+
}
|
|
300951
301402
|
for (const sourceInfo of sourceInfos) {
|
|
300952
301403
|
const sourceName = sourceInfo.name;
|
|
300953
|
-
const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
|
|
300954
301404
|
const governanceKey = sourceContextKey(modelPath, sourceName);
|
|
301405
|
+
if (droppedSources.has(governanceKey)) {
|
|
301406
|
+
continue;
|
|
301407
|
+
}
|
|
301408
|
+
const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
|
|
300955
301409
|
if (!governance.has(governanceKey)) {
|
|
300956
301410
|
const apiSource = apiSources.find((c) => c.name === sourceName);
|
|
300957
301411
|
if (apiSource) {
|
|
@@ -300964,6 +301418,10 @@ async function collectEntities(pkg) {
|
|
|
300964
301418
|
...given.default !== undefined ? { default: given.default } : {}
|
|
300965
301419
|
}
|
|
300966
301420
|
] : []),
|
|
301421
|
+
accessFilter: (apiSource.accessFilter ?? []).map((expression) => ({
|
|
301422
|
+
expression,
|
|
301423
|
+
given_names: referencedGivenNames(expression)
|
|
301424
|
+
})),
|
|
300967
301425
|
authorize: (apiSource.authorize ?? []).map((expression) => ({
|
|
300968
301426
|
expression,
|
|
300969
301427
|
given_names: referencedGivenNames(expression)
|
|
@@ -301038,6 +301496,9 @@ async function collectEntities(pkg) {
|
|
|
301038
301496
|
continue;
|
|
301039
301497
|
if (!query.sourceName)
|
|
301040
301498
|
continue;
|
|
301499
|
+
if (droppedSources.has(sourceContextKey(modelPath, query.sourceName))) {
|
|
301500
|
+
continue;
|
|
301501
|
+
}
|
|
301041
301502
|
entities.push({
|
|
301042
301503
|
id: String(n++),
|
|
301043
301504
|
kind: "query",
|
|
@@ -301057,7 +301518,7 @@ async function collectEntities(pkg) {
|
|
|
301057
301518
|
seen.add(key);
|
|
301058
301519
|
return true;
|
|
301059
301520
|
});
|
|
301060
|
-
return { entities: collapseAliases(deduped), governance };
|
|
301521
|
+
return { entities: collapseAliases(deduped), governance, droppedSources };
|
|
301061
301522
|
}
|
|
301062
301523
|
function malloyType(field) {
|
|
301063
301524
|
const kind = field.type?.kind;
|
|
@@ -301160,6 +301621,7 @@ function buildSourceContext(collected) {
|
|
|
301160
301621
|
joins: [],
|
|
301161
301622
|
...summary ? { oneLineSummary: summary } : {},
|
|
301162
301623
|
...gates?.givens.length ? { givens: gates.givens } : {},
|
|
301624
|
+
...gates?.accessFilter.length ? { accessFilter: gates.accessFilter } : {},
|
|
301163
301625
|
...gates?.authorize.length ? { authorize: gates.authorize } : {},
|
|
301164
301626
|
...gates?.filters.length ? { filters: gates.filters } : {}
|
|
301165
301627
|
});
|
|
@@ -301207,7 +301669,8 @@ async function getPackageIndex(environmentStore, environmentName, packageName) {
|
|
|
301207
301669
|
byId,
|
|
301208
301670
|
index,
|
|
301209
301671
|
entityCount: entities.length,
|
|
301210
|
-
sourceContext: buildSourceContext(collected)
|
|
301672
|
+
sourceContext: buildSourceContext(collected),
|
|
301673
|
+
droppedSources: collected.droppedSources
|
|
301211
301674
|
};
|
|
301212
301675
|
indexCache.set(pkg, built);
|
|
301213
301676
|
logger.debug("[MCP Tool getContext] Built and cached entity index", {
|
|
@@ -301225,13 +301688,13 @@ var GET_CONTEXT_DESCRIPTION = `Retrieve the entities in a Malloy package most re
|
|
|
301225
301688
|
- Read warnings and any error/stale field before trusting a number.
|
|
301226
301689
|
- A source's joins list is complete: empty means it declares none, so write that relationship inline.
|
|
301227
301690
|
- Read a source's doc before querying: it carries grain and population rules its fields do not.
|
|
301228
|
-
- authorize
|
|
301691
|
+
- accessFilter/authorize mean gated; a deny-all source never appears here.
|
|
301229
301692
|
|
|
301230
301693
|
## Parameters
|
|
301231
301694
|
search_targets: one per concept, {target_type, search_text}; target_type is source|dimension|measure|view|join|dimensional_value, omitting search_text enumerates that type. scopes: {environment, package} + optional model_path, source, entity_name. limit caps sources (max 150, counted as cards). offset pages a listing. user_prompt: the question asked. include_code or a pinned entity_name returns code.
|
|
301232
301695
|
|
|
301233
301696
|
## Response
|
|
301234
|
-
sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, authorize
|
|
301697
|
+
sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, accessFilter/authorize, filter_params. entities[] nest under it: name, entity_type, description, data_type, relationship (fan-out), join_path, aliases, relevance, entity_id. A joined field's name IS its dotted path; use it verbatim.
|
|
301235
301698
|
ranking, returned of total_available sources, next_offset on a listing, warnings[].
|
|
301236
301699
|
Semantic fills relevance: no sources = nothing cleared the floor; below_cutoff_count of total_entities rejected. "lexical" adds retrieval_reason; only "indexing" is worth a retry.
|
|
301237
301700
|
|
|
@@ -301266,7 +301729,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301266
301729
|
});
|
|
301267
301730
|
return contextError(buildMalloyUri({ environment: environmentName, package: packageName }, "get-context"), `${environmentName}/${packageName}`, error);
|
|
301268
301731
|
}
|
|
301269
|
-
const { byId, index, sourceContext } = pkgIndex;
|
|
301732
|
+
const { byId, index, sourceContext, droppedSources } = pkgIndex;
|
|
301270
301733
|
const uri = buildMalloyUri({ environment: environmentName, package: packageName }, "get-context");
|
|
301271
301734
|
let staleNote;
|
|
301272
301735
|
try {
|
|
@@ -301304,7 +301767,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301304
301767
|
});
|
|
301305
301768
|
const spendable = (rows) => sourceName ? rows.filter((e) => e.kind !== "source").length : rows.length;
|
|
301306
301769
|
const cappable = spendable(inScope);
|
|
301307
|
-
const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode);
|
|
301770
|
+
const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode, droppedSources);
|
|
301308
301771
|
const consumed = request.offset + sources2.length;
|
|
301309
301772
|
const cardsInScope = sourceName ? new Set(inScope.map((e) => sourceContextKey(e.modelPath, e.source ?? ""))).size : inScope.length;
|
|
301310
301773
|
const listingEnvelope = {
|
|
@@ -301420,7 +301883,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301420
301883
|
environmentName,
|
|
301421
301884
|
packageName,
|
|
301422
301885
|
searchTexts: searchTextsByIndex,
|
|
301423
|
-
includeCode: request.includeCode
|
|
301886
|
+
includeCode: request.includeCode,
|
|
301887
|
+
droppedSources
|
|
301424
301888
|
});
|
|
301425
301889
|
return jsonResource(uri, {
|
|
301426
301890
|
sources: sources2,
|
|
@@ -301481,7 +301945,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301481
301945
|
environmentName,
|
|
301482
301946
|
packageName,
|
|
301483
301947
|
searchTexts: searchTextsByIndex,
|
|
301484
|
-
includeCode: request.includeCode
|
|
301948
|
+
includeCode: request.includeCode,
|
|
301949
|
+
droppedSources
|
|
301485
301950
|
});
|
|
301486
301951
|
const envelope = {
|
|
301487
301952
|
sources,
|
|
@@ -301747,7 +302212,7 @@ class QueryController {
|
|
|
301747
302212
|
const p = await environment.getPackage(packageName, false);
|
|
301748
302213
|
const model = p.getModel(modelPath);
|
|
301749
302214
|
if (!model) {
|
|
301750
|
-
throw new ModelNotFoundError(
|
|
302215
|
+
throw new ModelNotFoundError(`No queryable model "${modelPath}".`);
|
|
301751
302216
|
} else {
|
|
301752
302217
|
const {
|
|
301753
302218
|
result,
|
|
@@ -309865,6 +310330,23 @@ class Package {
|
|
|
309865
310330
|
const explores = this.packageMetadata.explores;
|
|
309866
310331
|
return !!(explores && explores.length > 0);
|
|
309867
310332
|
}
|
|
310333
|
+
surfaceWidenedWarning;
|
|
310334
|
+
noteSurfaceChangeFrom(previousSurface) {
|
|
310335
|
+
this.surfaceWidenedWarning = undefined;
|
|
310336
|
+
if (!previousSurface || previousSurface.length === 0 || this.packageMetadata.explores !== undefined) {
|
|
310337
|
+
return;
|
|
310338
|
+
}
|
|
310339
|
+
const message = `This package published "${previousSurface.join('", "')}" before the last ` + `reload and publishes no surface now, so every model in it is listed ` + `and queryable by name again, including the sources that surface was ` + `withholding. A surface file that was deleted or renamed is the usual ` + `cause. If that was intended, nothing to do. If not, restore the file ` + `(under its original name -- the name IS the surface), or declare an ` + `"explores" in publisher.json naming what this package should ` + `publish. To keep the package open deliberately, write ` + `"explores": [], which says so and stops this notice.`;
|
|
310340
|
+
this.surfaceWidenedWarning = message;
|
|
310341
|
+
logger.warn(`Package ${this.packageName} no longer publishes a surface`, {
|
|
310342
|
+
packageName: this.packageName,
|
|
310343
|
+
detail: message
|
|
310344
|
+
});
|
|
310345
|
+
}
|
|
310346
|
+
surfaceIsIndexModel() {
|
|
310347
|
+
const explores = this.packageMetadata.explores;
|
|
310348
|
+
return this.models.has(INDEX_MODEL_NAME) && explores?.length === 1 && explores[0] === INDEX_MODEL_NAME;
|
|
310349
|
+
}
|
|
309868
310350
|
exploreSet() {
|
|
309869
310351
|
const explores = this.packageMetadata.explores;
|
|
309870
310352
|
return explores && explores.length > 0 ? new Set(explores) : null;
|
|
@@ -310204,6 +310686,8 @@ class Package {
|
|
|
310204
310686
|
...this.droppedPersistWarnings(),
|
|
310205
310687
|
...this.persistRefusalWarnings(),
|
|
310206
310688
|
...this.emptyDiscoveryWarnings(),
|
|
310689
|
+
...this.brokenSurfaceWarnings(),
|
|
310690
|
+
...this.surfaceWidenedWarning !== undefined ? [{ message: this.surfaceWidenedWarning }] : [],
|
|
310207
310691
|
...this.persistenceCollisionWarnings().map((message) => ({ message })),
|
|
310208
310692
|
...this.preaggregateAccessWarnings(),
|
|
310209
310693
|
...incrementalPolicyAdvisories(this.incrementalPolicySources),
|
|
@@ -310364,7 +310848,7 @@ class Package {
|
|
|
310364
310848
|
if (entry.endsWith(NOTEBOOK_FILE_SUFFIX)) {
|
|
310365
310849
|
problems.push({
|
|
310366
310850
|
entry,
|
|
310367
|
-
reason: `notebooks are always
|
|
310851
|
+
reason: `notebooks are always listed and cannot be explores. ` + `Fix: remove it, and list a ${MODEL_FILE_SUFFIX} model file instead.`
|
|
310368
310852
|
});
|
|
310369
310853
|
} else if (!malloyModels.has(entry)) {
|
|
310370
310854
|
problems.push({
|
|
@@ -310535,13 +311019,39 @@ class Package {
|
|
|
310535
311019
|
if (model.hasEmptyDiscoverySurface()) {
|
|
310536
311020
|
warnings.push({
|
|
310537
311021
|
model: modelPath,
|
|
310538
|
-
message: `Model "${modelPath}" is
|
|
311022
|
+
message: `Model "${modelPath}" is on this package's discovery ` + `surface but exposes nothing: its export closure surfaces ` + `no sources or named queries (typically an import-only ` + `file, or an export {} that filters everything out). Add ` + `e.g. 'export { source_name }' to surface sources on this ` + `model, or ` + (this.surfaceIsIndexModel() ? `delete the file AND any "explores" entry naming it. ` + `Deleting the file alone returns the package to ` + `listing every model only when no "explores" was ` + `written; where one was, it would be left naming a ` + `model that no longer exists and the package would ` + `list nothing.` : `remove it from explores.`)
|
|
310539
311023
|
});
|
|
310540
311024
|
}
|
|
310541
311025
|
}
|
|
310542
311026
|
return warnings;
|
|
310543
311027
|
}
|
|
311028
|
+
brokenSurfaceWarnings() {
|
|
311029
|
+
const exploreSet = this.exploreSet();
|
|
311030
|
+
if (!exploreSet || exploreSet.size === 0)
|
|
311031
|
+
return [];
|
|
311032
|
+
const surface = Array.from(this.models.entries()).filter(([modelPath]) => modelPath.endsWith(MODEL_FILE_SUFFIX) && exploreSet.has(modelPath));
|
|
311033
|
+
if (surface.length === 0)
|
|
311034
|
+
return [];
|
|
311035
|
+
if (!surface.every(([, model]) => !!model.getCompilationError())) {
|
|
311036
|
+
return [];
|
|
311037
|
+
}
|
|
311038
|
+
const collateral = Array.from(this.models.entries()).filter(([modelPath, model]) => modelPath.endsWith(MODEL_FILE_SUFFIX) && !exploreSet.has(modelPath) && !model.getCompilationError()).length;
|
|
311039
|
+
const [firstPath] = surface[0];
|
|
311040
|
+
const subject = surface.length === 1 ? `Model "${firstPath}" is this package's whole discovery surface ` + `and failed to compile` : `Every model on this package's discovery surface ` + `(${surface.map(([p]) => `"${p}"`).join(", ")}) failed to ` + `compile`;
|
|
311041
|
+
return [
|
|
311042
|
+
{
|
|
311043
|
+
model: firstPath,
|
|
311044
|
+
message: `${subject}, so the package exposes nothing: every model in ` + `it, including the ${collateral} that compiled, is now refused ` + `by name with a 404 that reads as "does not exist". Fix the ` + `compile error${surface.length === 1 ? "" : "s"} to restore ` + `them -- ` + surface.map(([modelPath, model]) => `${modelPath}: ` + `${model.getCompilationError()?.message ?? "unknown error"}`).join("; ")
|
|
311045
|
+
}
|
|
311046
|
+
];
|
|
311047
|
+
}
|
|
310544
311048
|
logEmptyDiscoveryWarnings() {
|
|
311049
|
+
for (const warning of this.brokenSurfaceWarnings()) {
|
|
311050
|
+
logger.warn(`Package ${this.packageName} has a broken discovery surface`, {
|
|
311051
|
+
packageName: this.packageName,
|
|
311052
|
+
detail: warning.message
|
|
311053
|
+
});
|
|
311054
|
+
}
|
|
310545
311055
|
for (const warning of this.emptyDiscoveryWarnings()) {
|
|
310546
311056
|
logger.warn(`Package ${this.packageName} has a blank-looking model`, {
|
|
310547
311057
|
packageName: this.packageName,
|
|
@@ -310659,7 +311169,9 @@ class Package {
|
|
|
310659
311169
|
await this.pushPreaggregateServeModels();
|
|
310660
311170
|
this.renderTagWarnings = renderTagWarnings;
|
|
310661
311171
|
this.manifestWarnings = outcome.packageMetadata.manifestWarnings ?? [];
|
|
311172
|
+
const previousSurface = this.packageMetadata.explores;
|
|
310662
311173
|
this.packageMetadata.explores = outcome.packageMetadata.explores;
|
|
311174
|
+
this.noteSurfaceChangeFrom(previousSurface);
|
|
310663
311175
|
this.packageMetadata.queryableSources = outcome.packageMetadata.queryableSources;
|
|
310664
311176
|
this.packageMetadata.manifestLocation = outcome.packageMetadata.manifestLocation ?? null;
|
|
310665
311177
|
this.applyDiscoveryPolicyToModels();
|
|
@@ -310821,6 +311333,29 @@ class Package {
|
|
|
310821
311333
|
});
|
|
310822
311334
|
}
|
|
310823
311335
|
}
|
|
311336
|
+
lintTilesAgainstSurface(modelPath, manifest) {
|
|
311337
|
+
const model = this.models.get(modelPath);
|
|
311338
|
+
if (!model || !manifest.tiles)
|
|
311339
|
+
return [];
|
|
311340
|
+
const listedModel = (this.packageMetadata.explores ?? []).find((entry) => entry.endsWith(MODEL_FILE_SUFFIX) && !isDashboardModelPath(entry));
|
|
311341
|
+
const importInto = listedModel ? `a listed file such as "${listedModel}"` : `a file you list there`;
|
|
311342
|
+
const findings = [];
|
|
311343
|
+
for (const tile of manifest.tiles) {
|
|
311344
|
+
try {
|
|
311345
|
+
model.assertQueryBoundaryEarly(undefined, undefined, `run: ${tile.query}`);
|
|
311346
|
+
} catch (error) {
|
|
311347
|
+
if (!(error instanceof NotQueryableError))
|
|
311348
|
+
throw error;
|
|
311349
|
+
findings.push({
|
|
311350
|
+
model: modelPath,
|
|
311351
|
+
subject: manifest.name,
|
|
311352
|
+
message: `tile "${tile.query}" reads a source this package's surface ` + `does not publish, so the tile answers 404 once served, ` + `although the file compiles. Listing a file publishes what ` + `it declares, not what it imports. Fix: add the file that ` + `declares the source to 'explores', keeping the entries ` + `already there, or import it into ${importInto} and add ` + `it to that file's export { … }.`,
|
|
311353
|
+
severity: "error"
|
|
311354
|
+
});
|
|
311355
|
+
}
|
|
311356
|
+
}
|
|
311357
|
+
return findings;
|
|
311358
|
+
}
|
|
310824
311359
|
async lintDashboards(factsByPath, allFacts, heldBack, unconventionalSlugs, knownSlugs, droppedByError) {
|
|
310825
311360
|
const warnings = [];
|
|
310826
311361
|
const withheldDrills = new Map;
|
|
@@ -310854,10 +311389,11 @@ class Package {
|
|
|
310854
311389
|
});
|
|
310855
311390
|
}
|
|
310856
311391
|
}
|
|
311392
|
+
const remedy = this.surfaceIsIndexModel() ? `This package's surface is "${INDEX_MODEL_NAME}", and a ` + `dashboard file cannot be exported from it -- dashboards are ` + `files, not sources. To serve this one, declare (or extend) ` + `an 'explores' in publisher.json listing both ` + `"${INDEX_MODEL_NAME}" and "${modelPath}"; an explicit key ` + `overrides the convention. Or set ` + `queryableSources: "all" to keep the curated surface for ` + `discovery only.` : `Add it to 'explores', or set queryableSources: "all" to ` + `keep the curated surface for discovery only.`;
|
|
310857
311393
|
warnings.push({
|
|
310858
311394
|
model: modelPath,
|
|
310859
311395
|
subject: name,
|
|
310860
|
-
message: `is a dashboard, but "${modelPath}" is not
|
|
311396
|
+
message: `is a dashboard, but "${modelPath}" is not part of this ` + `package's discovery surface and this package sets ` + `queryableSources: "declared", so its query would be ` + `refused. It is not served. ${remedy} Listing it is not ` + `always sufficient on its own: the queryable sources are ` + `the union of every listed file's export closure, so a tile ` + `reading a source that only an UNLISTED file exports is ` + `still refused. List that file too, or re-export the source ` + `from one already listed.`,
|
|
310861
311397
|
severity: "warn"
|
|
310862
311398
|
});
|
|
310863
311399
|
}
|
|
@@ -310867,6 +311403,11 @@ class Package {
|
|
|
310867
311403
|
for (const finding of findings) {
|
|
310868
311404
|
warnings.push({ model: modelPath, ...finding });
|
|
310869
311405
|
}
|
|
311406
|
+
if (manifest) {
|
|
311407
|
+
for (const finding of this.lintTilesAgainstSurface(modelPath, manifest)) {
|
|
311408
|
+
warnings.push(finding);
|
|
311409
|
+
}
|
|
311410
|
+
}
|
|
310870
311411
|
}
|
|
310871
311412
|
const drillFacts = Array.from(allFacts.values());
|
|
310872
311413
|
for (const finding of withheldDrills.values()) {
|
|
@@ -311741,6 +312282,7 @@ ${source}` : source ?? "";
|
|
|
311741
312282
|
await this.rebindServeBindingsFromLocalStore(_package);
|
|
311742
312283
|
if (existingPackage !== undefined && reload) {
|
|
311743
312284
|
this.retireConnectionGeneration(`package ${packageName}`, () => existingPackage.getMalloyConfig().shutdown("close"));
|
|
312285
|
+
_package.noteSurfaceChangeFrom(existingPackage.getPackageMetadata().explores);
|
|
311744
312286
|
}
|
|
311745
312287
|
this.packages.set(packageName, _package);
|
|
311746
312288
|
this.setPackageStatus(packageName, "serving" /* SERVING */);
|
|
@@ -312044,11 +312586,12 @@ ${source}` : source ?? "";
|
|
|
312044
312586
|
const materializationBase = metadata.materialization !== undefined ? { ...metadata.materialization, ...preservedQueryMetadata } : onDiskMaterialization !== undefined ? { ...onDiskMaterialization } : undefined;
|
|
312045
312587
|
const materializationBlock = resolvedScope !== undefined ? { ...materializationBase ?? {}, scope: resolvedScope } : materializationBase;
|
|
312046
312588
|
const resolvedQueryMetadata = metadata.queryMetadata ?? materializationBlock?.queryMetadata ?? existingManifest.queryMetadata;
|
|
312589
|
+
const echoesDerivedSurface = existingManifest.explores === undefined && Array.isArray(metadata.explores) && metadata.explores.length === 1 && metadata.explores[0] === INDEX_MODEL_NAME;
|
|
312047
312590
|
const updatedManifest = {
|
|
312048
312591
|
...existingManifest,
|
|
312049
312592
|
name: metadata.name,
|
|
312050
312593
|
description: metadata.description,
|
|
312051
|
-
...metadata.explores !== undefined ? { explores: metadata.explores } : {},
|
|
312594
|
+
...metadata.explores !== undefined && !echoesDerivedSurface ? { explores: metadata.explores } : {},
|
|
312052
312595
|
...metadata.queryableSources !== undefined ? { queryableSources: metadata.queryableSources } : {},
|
|
312053
312596
|
...metadata.manifestLocation !== undefined ? { manifestLocation: metadata.manifestLocation } : {},
|
|
312054
312597
|
...resolvedScope !== undefined ? { scope: resolvedScope } : {},
|
|
@@ -313578,9 +314121,9 @@ class EnvironmentStore {
|
|
|
313578
314121
|
}
|
|
313579
314122
|
const file = fs9.createWriteStream(zipFilePath);
|
|
313580
314123
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
313581
|
-
await new Promise((
|
|
314124
|
+
await new Promise((resolve6, reject2) => {
|
|
313582
314125
|
file.on("error", reject2);
|
|
313583
|
-
file.on("finish",
|
|
314126
|
+
file.on("finish", resolve6);
|
|
313584
314127
|
});
|
|
313585
314128
|
await this.unzipEnvironment(zipFilePath);
|
|
313586
314129
|
logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -313618,9 +314161,9 @@ class EnvironmentStore {
|
|
|
313618
314161
|
}
|
|
313619
314162
|
const file = fs9.createWriteStream(absoluteFilePath);
|
|
313620
314163
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
313621
|
-
await new Promise((
|
|
314164
|
+
await new Promise((resolve6, reject2) => {
|
|
313622
314165
|
file.on("error", reject2);
|
|
313623
|
-
file.on("finish",
|
|
314166
|
+
file.on("finish", resolve6);
|
|
313624
314167
|
});
|
|
313625
314168
|
}));
|
|
313626
314169
|
logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -313640,7 +314183,7 @@ class EnvironmentStore {
|
|
|
313640
314183
|
await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
313641
314184
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
313642
314185
|
const reporter = new CloneProgressReporter(cloneProgressLabel(`${owner}/${repoName}`, progressContext));
|
|
313643
|
-
await new Promise((
|
|
314186
|
+
await new Promise((resolve6, reject2) => {
|
|
313644
314187
|
esm_default2({
|
|
313645
314188
|
progress: (event) => reporter.onProgress(event)
|
|
313646
314189
|
}).clone(repoUrl, absoluteDirPath, GIT_CLONE_OPTIONS, (err) => {
|
|
@@ -313655,7 +314198,7 @@ class EnvironmentStore {
|
|
|
313655
314198
|
reject2(err);
|
|
313656
314199
|
return;
|
|
313657
314200
|
}
|
|
313658
|
-
|
|
314201
|
+
resolve6();
|
|
313659
314202
|
});
|
|
313660
314203
|
});
|
|
313661
314204
|
if (!cleanPackagePath) {
|
|
@@ -314445,7 +314988,7 @@ class Protocol {
|
|
|
314445
314988
|
}
|
|
314446
314989
|
request(request, resultSchema, options) {
|
|
314447
314990
|
const { relatedRequestId, resumptionToken, onresumptiontoken } = options !== null && options !== undefined ? options : {};
|
|
314448
|
-
return new Promise((
|
|
314991
|
+
return new Promise((resolve6, reject2) => {
|
|
314449
314992
|
var _a2, _b, _c, _d, _e, _f;
|
|
314450
314993
|
if (!this._transport) {
|
|
314451
314994
|
reject2(new Error("Not connected"));
|
|
@@ -314496,7 +315039,7 @@ class Protocol {
|
|
|
314496
315039
|
}
|
|
314497
315040
|
try {
|
|
314498
315041
|
const result = resultSchema.parse(response.result);
|
|
314499
|
-
|
|
315042
|
+
resolve6(result);
|
|
314500
315043
|
} catch (error) {
|
|
314501
315044
|
reject2(error);
|
|
314502
315045
|
}
|
|
@@ -316647,17 +317190,17 @@ var compileShape = {
|
|
|
316647
317190
|
source: exports_external.string().optional().describe("The Malloy source to validate; never executed. Required at scope append/file; optional at scope package (a what-if replacement for modelPath)."),
|
|
316648
317191
|
scope: exports_external.enum(["append", "file", "package"]).optional().describe(`What source means. "append" (default): append to modelPath. "file": compile AS modelPath to validate an edit. "package": run reload's worker compiler over all .malloy/.malloynb files without serving the result; an optional source replaces modelPath.`),
|
|
316649
317192
|
includeSql: exports_external.boolean().optional().describe("When true and the source ends in a runnable query, also return the generated SQL for inspection. The query is still not run."),
|
|
316650
|
-
givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any
|
|
317193
|
+
givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any authorize gate on the target model, whether or not includeSql is set.")
|
|
316651
317194
|
};
|
|
316652
317195
|
var COMPILE_DESCRIPTION = `Compile-check Malloy without running a query. Use this while authoring instead of a throwaway execute_query.
|
|
316653
317196
|
|
|
316654
317197
|
## Scopes (the scope parameter)
|
|
316655
317198
|
- "append" (default): append source to modelPath. Use for NEW definitions; existing definitions report "Cannot redefine". Positions refer to the concatenated file.
|
|
316656
317199
|
- "file": compile source AS modelPath. Use to validate an EDIT before saving; positions match the submitted file.
|
|
316657
|
-
- "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and
|
|
317200
|
+
- "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and authorize gates still apply to caller text. A missing exact path is warned and treated as a new file. Save and call reload_package to serve a clean edit.
|
|
316658
317201
|
|
|
316659
317202
|
## Parameters
|
|
316660
|
-
- environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and
|
|
317203
|
+
- environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and authorize-gate values. Caller source may not declare an access-control gate — neither #(authorize) nor #(access_filter).
|
|
316661
317204
|
|
|
316662
317205
|
## Response
|
|
316663
317206
|
{ status: "success"|"error", diagnostics: [{ severity, message, code, model, line, character, endLine, endCharacter, replacement }], sql? }. Positions are 0-based; model is the package-relative file the diagnostic points at (which can be pre-existing content, not your source). status is "error" only when an error-severity diagnostic exists; errors are also stated in a plain text block.`;
|
|
@@ -318746,6 +319289,21 @@ If the agent could not reasonably have known to ask, that is a model gap.
|
|
|
318746
319289
|
| \`GUIDANCE-NOT-RETRIEVED\` | entities came back, governing guidance did not | model: put guidance on the entities agents search for |
|
|
318747
319290
|
| \`GUIDANCE-DECLINED\` | guidance was retrieved and judged inapplicable | model: state the business default, not a caveat |
|
|
318748
319291
|
|
|
319292
|
+
**Before diagnosing a \`NOT-RETURNED\`, check the call actually returned
|
|
319293
|
+
nothing.** The \`tool_call\` event carries \`retrieval_mode\` and \`rankedSummary\`.
|
|
319294
|
+
A \`rankedSummary\` of \`null\` means the response could not be read, not that it
|
|
319295
|
+
was empty -- a body too large for the model's context is written to a file, and
|
|
319296
|
+
the run records no summary for it. There is nothing to diagnose there: say the
|
|
319297
|
+
call was not measured.
|
|
319298
|
+
|
|
319299
|
+
And **never attribute a miss to the embedding index unless \`retrieval_mode\`
|
|
319300
|
+
says \`lexical\`.** The mode is recorded per call precisely so this is checkable.
|
|
319301
|
+
A diagnosis once explained a phantom empty result with "the index was not ready
|
|
319302
|
+
yet" on a run whose index was ready before the first question and whose
|
|
319303
|
+
response did hold results; the empty list was a parsing bug in the harness. An
|
|
319304
|
+
unverifiable cause that sounds right is worse than \`needs_human\`, because it
|
|
319305
|
+
closes the finding.
|
|
319306
|
+
|
|
318749
319307
|
A missing join is coverage, not an agent-call miss. The model has to volunteer
|
|
318750
319308
|
relationships. A declared join is not a retrieval entity; do not look for it in
|
|
318751
319309
|
\`get_context\` results.
|
|
@@ -319791,6 +320349,27 @@ Output, exactly this shape:
|
|
|
319791
320349
|
reading that produced it, that is \`no_match\`. Use \`near_match\` only when you
|
|
319792
320350
|
can name the rubric clause that makes the difference defensible.
|
|
319793
320351
|
|
|
320352
|
+
**A rubric clause cannot make a wrong VALUE defensible, and a clause that
|
|
320353
|
+
tries is a defect in the rubric rather than a licence to you.** \`near_match\`
|
|
320354
|
+
turns on the answer being right under a reading the QUESTION allows -- a tie
|
|
320355
|
+
broken the other way, a grain the question left open, a basis the question
|
|
320356
|
+
never fixed. It does not turn on the answer being transparent about how it
|
|
320357
|
+
got a figure the question did not ask for. Those two look alike in a rubric
|
|
320358
|
+
and are opposites in a report: one is a number a reader can act on, the
|
|
320359
|
+
other is a number a reader would act on wrongly. Naming the method makes a
|
|
320360
|
+
wrong figure DIAGNOSABLE, which is worth having, and it is not partial
|
|
320361
|
+
credit.
|
|
320362
|
+
|
|
320363
|
+
The test, before you write \`near_match\` on a case with a value: would a
|
|
320364
|
+
reader who acted on this figure be wrong? If yes, it is \`no_match\` however
|
|
320365
|
+
plainly the answer explained itself, and however the rubric is worded. Say
|
|
320366
|
+
in \`why\` that you are overriding a rubric clause, so the clause gets fixed.
|
|
320367
|
+
This rule exists because a set shipped one: a question asked for sales over
|
|
320368
|
+
the company's season, the answer gave the meteorological window 25% lower
|
|
320369
|
+
and said which window it used, and a clause granting \`near_match\` for a
|
|
320370
|
+
stated window kept a materially wrong answer out of the pass rate
|
|
320371
|
+
entirely -- the arm reported 100%.
|
|
320372
|
+
|
|
319794
320373
|
It is a third outcome because as a pass it was a large share of the measured
|
|
319795
320374
|
noise: the same unchanged answer reads \`match\` in one run and \`near_match\`
|
|
319796
320375
|
in the next, and the pass rate moves although nothing did. A verdict whose
|
|
@@ -320454,11 +321033,19 @@ the run measure something other than what it names:
|
|
|
320454
321033
|
first time somebody tidies up; the set, the run directory and the write-up
|
|
320455
321034
|
belong in git beside the model.
|
|
320456
321035
|
|
|
320457
|
-
2. The server must be up
|
|
320458
|
-
|
|
320459
|
-
|
|
320460
|
-
|
|
320461
|
-
|
|
321036
|
+
2. The server must be up. Where your host offers retrieval tracing, turn it on
|
|
321037
|
+
and confirm a trace lookup answers, so a call's ranked results can be
|
|
321038
|
+
recovered afterwards.
|
|
321039
|
+
|
|
321040
|
+
**Open-source Publisher has none.** \`PUBLISHER_MCP_TRACE=retrieval\` is set by
|
|
321041
|
+
\`serve.py\` and read by nothing: the trace store was written and never merged,
|
|
321042
|
+
so there is no trace tool and \`traceId\` is null on every local attempt. This
|
|
321043
|
+
does not block a scored run, because attribution never depended on it:
|
|
321044
|
+
\`rankedSummary\` is copied onto the \`tool_call\` event at capture, precisely so
|
|
321045
|
+
the evidence survives without a store. So do not refuse a local run for want
|
|
321046
|
+
of tracing -- an earlier version of this rule did, and it refused every local
|
|
321047
|
+
run there has ever been. Refuse one whose \`tool_call\` events carry no
|
|
321048
|
+
\`rankedSummary\`, which is what attribution actually reads.
|
|
320462
321049
|
|
|
320463
321050
|
3. Health-check: your host's status check until it reports serving, and inspect
|
|
320464
321051
|
\`loadErrors\`. A dead database that still answers HTTP is an environment
|
|
@@ -320626,6 +321213,13 @@ conductor. Do not:
|
|
|
320626
321213
|
|
|
320627
321214
|
- The model is the only thing improve edits. No question text, qids, or
|
|
320628
321215
|
expected values in any name, doc, or comment.
|
|
321216
|
+
- **You are measuring a model, not reviewing this harness.** Read a script when
|
|
321217
|
+
a number you have to report cannot be explained otherwise, and stop there. Do
|
|
321218
|
+
not audit the scripts, propose fixes to them, or hand back tooling critique
|
|
321219
|
+
in place of a result -- a run that ends in harness feedback has not answered
|
|
321220
|
+
the question it was asked. A defect that changed a number gets one sentence
|
|
321221
|
+
in the report; a defect that changed nothing gets one line in chat, once.
|
|
321222
|
+
Fixing it is a different task, and the user starts it.
|
|
320629
321223
|
- When the environment misbehaves, stop. Never diagnose a sick system. The
|
|
320630
321224
|
harness is part of the environment: a known-broken measurement does not
|
|
320631
321225
|
become quotable by being finished. Measured against this directive, an arm
|
|
@@ -321586,6 +322180,30 @@ one exception is anything that qualifies THIS run's number -- a truncated
|
|
|
321586
322180
|
attempt, a contaminated one, an unestablished answer key -- which the "Eval
|
|
321587
322181
|
failures" section exists for.
|
|
321588
322182
|
|
|
322183
|
+
**Do not audit the harness. You were asked to measure a model.** This is the
|
|
322184
|
+
most common way this job goes wrong, and it does not look like going wrong: a
|
|
322185
|
+
run turns up something odd in a script, the odd thing is genuinely a bug, and
|
|
322186
|
+
the reply comes back as a critique of the tooling with the model's score
|
|
322187
|
+
somewhere underneath. The reader asked what their model scored. Answer that.
|
|
322188
|
+
|
|
322189
|
+
So, unless the user asked you to work on the harness:
|
|
322190
|
+
|
|
322191
|
+
- Do not read harness source to satisfy your own curiosity about a number.
|
|
322192
|
+
Read it when a number you must report cannot be explained any other way, and
|
|
322193
|
+
stop when it can.
|
|
322194
|
+
- Do not propose harness fixes, refactors, flags or "while I was in there"
|
|
322195
|
+
improvements. Not in the report, not in the chat reply.
|
|
322196
|
+
- When a harness defect DID change this run's number, the report gets one
|
|
322197
|
+
sentence: what the number should be and why. Not the mechanism, not the
|
|
322198
|
+
file, not the fix.
|
|
322199
|
+
- Keep a defect that changed nothing out of the report entirely. Mention it
|
|
322200
|
+
once in chat, in a line, and let the user decide whether they want it
|
|
322201
|
+
chased.
|
|
322202
|
+
|
|
322203
|
+
A harness bug you found and did not chase is not a loose end. It is the job
|
|
322204
|
+
being done. If the user wants it fixed they will say so, and then it is a
|
|
322205
|
+
different task with its own turn.
|
|
322206
|
+
|
|
321589
322207
|
## Step 1: build the artifacts, before writing a word
|
|
321590
322208
|
|
|
321591
322209
|
\`\`\`bash
|
|
@@ -321624,6 +322242,29 @@ not in \`~/Downloads\`, both of which have lost a findings document before.
|
|
|
321624
322242
|
from the console.** A figure retyped from scrollback is a figure nobody can
|
|
321625
322243
|
check, and the console rounds.
|
|
321626
322244
|
|
|
322245
|
+
### Keep it under a page and a half
|
|
322246
|
+
|
|
322247
|
+
**Budget: about 80 lines, and 120 is the ceiling for a run with several
|
|
322248
|
+
distinct failures.** Reports have shipped at 200 and the length is not
|
|
322249
|
+
thoroughness -- it is the artifact links restated as prose, the cascade
|
|
322250
|
+
explained twice, and a paragraph apologising for a figure nobody disputed.
|
|
322251
|
+
A reader who wants the detail opens the case matrix, which is why step 1 builds
|
|
322252
|
+
it. What cannot be recovered from the artifacts is your judgement: what broke,
|
|
322253
|
+
why, and what to do. Spend the lines there.
|
|
322254
|
+
|
|
322255
|
+
What earns its place: the headline, one row per case, one short entry per
|
|
322256
|
+
failure, the retrieval numbers, and the next steps. What does not: restating a
|
|
322257
|
+
number you already gave, explaining what a cascade is before showing it,
|
|
322258
|
+
defending a decision nobody questioned, or any section whose content is that
|
|
322259
|
+
nothing happened -- except "Eval failures", where "None." is the point.
|
|
322260
|
+
|
|
322261
|
+
**Being brief is not being terse with the vocabulary.** The reader does not
|
|
322262
|
+
know what \`near_match\`, a cluster, recall or a holdout is, so the first time
|
|
322263
|
+
one appears, say what it means in the same sentence -- "\`near_match\`, which is
|
|
322264
|
+
excluded from the pass rate" -- and then use it. Cutting the explanations is
|
|
322265
|
+
how a short report becomes an unreadable one; cutting the restatements is how
|
|
322266
|
+
it becomes a good one.
|
|
322267
|
+
|
|
321627
322268
|
### The template
|
|
321628
322269
|
|
|
321629
322270
|
\`\`\`markdown
|
|
@@ -321637,7 +322278,7 @@ check, and the console rounds.
|
|
|
321637
322278
|
- Target \`<env>/<package>\`, model \`<modelPath>\`, pinned \`<modelSha or targetVersion>\`
|
|
321638
322279
|
- Answerer \`<model>\`, judge \`<model>\`, cap \`<maxTurns>\` turns
|
|
321639
322280
|
- Steps run: scrape/run, eval<, diagnose><, improve>
|
|
321640
|
-
- Cost $X answerer + $Y judge
|
|
322281
|
+
- Cost $X answerer + $Y judge, N turns median / N p90, N entities delivered per answer
|
|
321641
322282
|
|
|
321642
322283
|
## The result
|
|
321643
322284
|
|
|
@@ -321648,9 +322289,15 @@ check, and the console rounds.
|
|
|
321648
322289
|
|
|
321649
322290
|
[case matrix](<url>) - [aggregate tables](<url>)
|
|
321650
322291
|
|
|
321651
|
-
##
|
|
322292
|
+
## Coverage, retrieval, accuracy, cost
|
|
321652
322293
|
|
|
321653
|
-
|
|
322294
|
+
**Coverage N of M** -- how many questions the model can express an answer to at
|
|
322295
|
+
all. **Entity recall N%** -- one number, first, before the cascade. It is the share
|
|
322296
|
+
of the entities an answer needed that retrieval actually handed the agent, and
|
|
322297
|
+
it is the headline of this section for the same reason the pass rate is the
|
|
322298
|
+
headline of the last one. Then the cascade: Covered? -> Retrieved? -> Correct?,
|
|
322299
|
+
with the per-arm numbers under each. Say in one clause what recall counts, then
|
|
322300
|
+
give the number.
|
|
321654
322301
|
|
|
321655
322302
|
## Model failures
|
|
321656
322303
|
|
|
@@ -321679,10 +322326,10 @@ these in order and put every one that fires into the list, with its command:
|
|
|
321679
322326
|
|
|
321680
322327
|
| If the run shows | Then the next step is |
|
|
321681
322328
|
|---|---|
|
|
321682
|
-
| \`coverage: unmeasured\` | run
|
|
322329
|
+
| \`coverage: unmeasured\` | the run was given \`--no-coverage\`, or the measurement failed and said so. It is measured by default, so say in "Eval failures" that this score cannot tell a model gap from a bad answer, and re-run without the flag |
|
|
321683
322330
|
| \`goldenCheck: skipped\` or the set names no \`truthPackage\` | build one with \`init_truth_package.py\`; until then the goldens were derived through the model under test and certify themselves |
|
|
321684
322331
|
| any golden still \`provisional\` | re-derive and \`verify_goldens.py --promote\` |
|
|
321685
|
-
| a stale entity name warning | fix \`expectedEntities\`; it scores as a retrieval miss on every run until you do |
|
|
322332
|
+
| a stale entity name warning | fix \`expectedEntities\`; it scores as a retrieval miss on every run until you do. A next step, not a section: it goes in this list and nowhere else in the report |
|
|
321686
322333
|
| a passing case with recall below 1.0 | check whether \`required\` over-specifies one path |
|
|
321687
322334
|
| \`truncated\` non-empty | re-run those cases at a higher cap with \`--from\` |
|
|
321688
322335
|
| diagnose did not run | run it, or say the failures have no owner yet |
|
|
@@ -321725,8 +322372,23 @@ wrong answer. One entry each: what the answer said, what was right, and the
|
|
|
321725
322372
|
mechanism in one sentence. Do not write a cluster id here; write what it got
|
|
321726
322373
|
wrong.
|
|
321727
322374
|
|
|
321728
|
-
**Eval failures** are
|
|
321729
|
-
|
|
322375
|
+
**Eval failures** are the things that stopped the run measuring the model, and
|
|
322376
|
+
**only** those. The test is one question: *did this cost a verdict, or make one
|
|
322377
|
+
untrustworthy?* If no, it does not appear in the report at any length.
|
|
322378
|
+
|
|
322379
|
+
That rules out most of what is tempting to put here, and all of it has been put
|
|
322380
|
+
here on a real run: a \`-dirty\` model pin, a stale-entity-name warning that cost
|
|
322381
|
+
no case, a defect you hit in the harness and worked around, a setup step that
|
|
322382
|
+
took two tries, anything you would open with "worth knowing, though it changes
|
|
322383
|
+
no verdict". A reader wants to know what their model scored. Harness defects are
|
|
322384
|
+
real and belong in a harness issue, filed against the harness -- that is the
|
|
322385
|
+
skill's opening rule, and this section is where it gets broken.
|
|
322386
|
+
|
|
322387
|
+
The section is usually two words. "**None.**" is a complete and good answer, and
|
|
322388
|
+
a reader who sees it learns exactly what they need to. Do not pad it into a
|
|
322389
|
+
paragraph explaining the absence.
|
|
322390
|
+
|
|
322391
|
+
Report every one that DID occur, with its count and its qids, and say plainly
|
|
321730
322392
|
that these are NOT evidence about the model:
|
|
321731
322393
|
|
|
321732
322394
|
| What happened | How it reads in the ledger | Who fixes it |
|
|
@@ -321745,11 +322407,32 @@ harness prints \`INCOMPLETE\` and withholds the percentage; the report does the
|
|
|
321745
322407
|
same. Give the counts and the re-run command instead of a number with a caveat,
|
|
321746
322408
|
because the number is what gets repeated and the caveat is what gets dropped.
|
|
321747
322409
|
|
|
321748
|
-
##
|
|
322410
|
+
## The four measurements, and why a report carries all of them
|
|
322411
|
+
|
|
322412
|
+
A pass rate says an answer was wrong. It does not say where, and on its own it
|
|
322413
|
+
makes every failure look like the model's. A run measures four things, in this
|
|
322414
|
+
order, and each one tells you whether the next is even a fair question:
|
|
322415
|
+
|
|
322416
|
+
| | Measures | A failure here means |
|
|
322417
|
+
|---|---|---|
|
|
322418
|
+
| **a. Coverage** | can the data and the model express an answer at all | nothing downstream was winnable. Fix the model |
|
|
322419
|
+
| **b. Retrieval** | did \`get_context\` hand the agent the entities it needed | the model has it and the agent never saw it: the docs, or the search wording |
|
|
322420
|
+
| **c. Accuracy** | did the agent get the answer right | it had what it needed and still missed: the agent, or a doc that misleads |
|
|
322421
|
+
| **d. Cost** | dollars, turns, wall-clock, entities per answer | it works and cannot be afforded, which is its own kind of not working |
|
|
321749
322422
|
|
|
321750
|
-
|
|
321751
|
-
|
|
321752
|
-
|
|
322423
|
+
**Coverage first, and a report without it is incomplete.** A model that cannot
|
|
322424
|
+
express an answer can never succeed at that question, so a pass rate quoted
|
|
322425
|
+
without coverage cannot distinguish a bad model from an unanswerable set.
|
|
322426
|
+
\`run_baseline.py\` measures it by default; if a run skipped it, say so in
|
|
322427
|
+
"Eval failures" and treat the score as provisional.
|
|
322428
|
+
|
|
322429
|
+
**Cost is a result, not an aside.** Report dollars, the turn and call counts,
|
|
322430
|
+
and the entity payload per answer. An agent that answers correctly in 40 turns
|
|
322431
|
+
and 75 delivered entities per question is a different product from one that
|
|
322432
|
+
does it in 8 and 5, and only the report says which you have.
|
|
322433
|
+
|
|
322434
|
+
Report them all, because a report that omits them makes every failure look
|
|
322435
|
+
like the model's:
|
|
321753
322436
|
|
|
321754
322437
|
**Copy the cascade the run printed. Do not re-derive it.** \`cascade_lines()\`
|
|
321755
322438
|
in \`run_baseline.py\` already renders every rung, and it carries one field a
|
|
@@ -321965,6 +322648,7 @@ This table is a catalogue of what exists, not of what is loaded. A host that ins
|
|
|
321965
322648
|
| \`skill:malloy-notebooks\` | Building Malloy notebooks (.malloynb) |
|
|
321966
322649
|
| \`skill:malloy-analysis-report\` | Combining validated queries into a notebook report or dashboard |
|
|
321967
322650
|
| \`skill:malloy-analysis-pitfalls\` | Checking a query and its results before presenting an answer |
|
|
322651
|
+
| \`skill:malloy-source-unreachable\` | A source is missing from discovery, or a query was refused with a 404 or 403 |
|
|
321968
322652
|
| \`malloy-notebook-chat\` | The chat is bound to a notebook or saved report; answer from its cells. Ships in \`analysis\`. |
|
|
321969
322653
|
| \`skill:malloy-phrase-detection\` | Turning a plain-English question into search targets for the context tool |
|
|
321970
322654
|
|
|
@@ -322021,7 +322705,7 @@ Call \`search_malloy_docs\` with your question. Use \`skill:malloy-patterns\` to
|
|
|
322021
322705
|
},
|
|
322022
322706
|
{
|
|
322023
322707
|
name: "malloy-analysis",
|
|
322024
|
-
description: "
|
|
322708
|
+
description: "Read before your first get_context call on any business or data question. The required workflow, not background: which source, which rows, how to verify. Skipping it causes confident wrong answers.",
|
|
322025
322709
|
body: `# Malloy analysis workflow
|
|
322026
322710
|
|
|
322027
322711
|
> **Tool names** are written bare here - \`get_context\`, \`execute_query\`, \`search_malloy_docs\`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
|
|
@@ -322131,7 +322815,7 @@ End with a short **Next steps**: one or two specific deeper analyses the data co
|
|
|
322131
322815
|
{
|
|
322132
322816
|
name: "malloy-analysis-pitfalls",
|
|
322133
322817
|
description: "Checks to run on a query and its numbers before a result is presented - wrong grain, fan-out, invented field names, mismatched filters, count vs distinct. The common data-analysis pitfalls.",
|
|
322134
|
-
body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats
|
|
322818
|
+
body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats is null }\n}\n```\n\nAn agent once refused a correct answer it had already computed because `count(aircraft.tail_num)` returned 36 against 17,875 flights; measured at the base grain, zero flights were missing a seat value.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
|
|
322135
322819
|
},
|
|
322136
322820
|
{
|
|
322137
322821
|
name: "malloy-analysis-report",
|
|
@@ -322430,6 +323114,13 @@ Scanned at a glance is a dashboard; read top to bottom is a notebook.
|
|
|
322430
323114
|
|
|
322431
323115
|
## Build sequence
|
|
322432
323116
|
|
|
323117
|
+
0. **CHECK THE PACKAGE CAN SERVE A DASHBOARD AT ALL.** If its root holds an \`index.malloy\`, that
|
|
323118
|
+
file is the package's surface, and a dashboard's file is not something it can \`export\`, so the
|
|
323119
|
+
dashboard you are about to write will compile and then be withheld. Every package made by the
|
|
323120
|
+
scaffolder has that file, so this is the common case, not the corner one. Serving dashboards
|
|
323121
|
+
there means declaring an \`explores\` in \`publisher.json\` naming \`index.malloy\` and every dashboard
|
|
323122
|
+
file, which overrides the convention. Settle that first: it is a different curation shape for the
|
|
323123
|
+
package, not a line to add at the end. See "Read the lint" for the warning it produces otherwise.
|
|
322433
323124
|
1. **READ THE MODEL FIRST.** Get the real source, view, dimension, and given names from the package:
|
|
322434
323125
|
\`get_context\` if you have it, otherwise the REST model endpoint or the \`.malloy\` files.
|
|
322435
323126
|
Never guess a name. A guessed field in a query fails the whole package load, not just that one
|
|
@@ -322840,8 +323531,8 @@ Package warnings after a reload are the dashboard's test suite. Fix all of them:
|
|
|
322840
323531
|
Findings carry a \`severity\`, but \`warn\` is the ordinary default and tells you nothing about how bad
|
|
322841
323532
|
one is. Read the text, not the severity and not the count. One message is worth recognising because
|
|
322842
323533
|
it changes what the rest of the list means: **"Dashboard lint stopped early, so this list is
|
|
322843
|
-
incomplete"**. A dashboard withheld from
|
|
322844
|
-
withheld file is not a clean bill of health.
|
|
323534
|
+
incomplete"**. A dashboard withheld from the package's surface also loses its own findings, so a
|
|
323535
|
+
short list for a withheld file is not a clean bill of health.
|
|
322845
323536
|
|
|
322846
323537
|
**Read the status the reload itself returns, not the listing.** One dashboard that fails to compile
|
|
322847
323538
|
fails the whole package load, and the reload answers **424** with the compile error. A package that
|
|
@@ -322856,11 +323547,22 @@ at all appears there too, without \`stale\`, and is absent from the listing enti
|
|
|
322856
323547
|
|
|
322857
323548
|
If the reload is 200 and the others are listed but yours is not, discovery skipped the file instead,
|
|
322858
323549
|
usually a missing or misspelled \`# artifact\` tag, which is the same mechanism that deliberately skips
|
|
322859
|
-
an untagged shared include. There is a second cause
|
|
322860
|
-
|
|
322861
|
-
|
|
322862
|
-
|
|
322863
|
-
|
|
323550
|
+
an untagged shared include. There is a second cause whenever the package has a curated surface: a
|
|
323551
|
+
dashboard whose file is off it is withheld rather than served, and the warning says so and names the
|
|
323552
|
+
fix. A package has a surface in two ways, and the second is easy to miss because no manifest field
|
|
323553
|
+
records it:
|
|
323554
|
+
|
|
323555
|
+
- the package's \`publisher.json\` carries an \`explores\` list, and the dashboard's file is not on it;
|
|
323556
|
+
- the package root holds an **\`index.malloy\`**, which IS the surface. This bites in both
|
|
323557
|
+
directions, and a dashboard's file is not something an \`index.malloy\` exports either way: adding
|
|
323558
|
+
that file to a package that has dashboards withholds every one of them, and adding a dashboard to
|
|
323559
|
+
a package that already has one (every scaffolded package does) means it is never served. The fix
|
|
323560
|
+
is the same in both directions, and it is not an edit to \`index.malloy\`: declare an \`explores\` in
|
|
323561
|
+
\`publisher.json\` naming \`index.malloy\` and every dashboard file.
|
|
323562
|
+
|
|
323563
|
+
Either way the surface is what matters, not the \`queryableSources\` setting, which is \`declared\` by
|
|
323564
|
+
default; a package with neither withholds nothing. Where there is a surface, a \`suggest\` source has
|
|
323565
|
+
to be queryable as well as resolvable, so it needs to be reachable through it too.
|
|
322864
323566
|
|
|
322865
323567
|
**A clean reload is not proof the tags are right.** The checks above read names and resolve them; the
|
|
322866
323568
|
separate warning for a tag that does not _parse_ is syntax only: it carries no
|
|
@@ -323229,7 +323931,7 @@ A confirmed source architecture and a confirmed set of field definitions (rename
|
|
|
323229
323931
|
{
|
|
323230
323932
|
name: "malloy-gotchas-queries",
|
|
323231
323933
|
description: "Common Malloy query and view mistakes, for writing views, queries or notebooks - chart constraints, aggregate filters, joined field aliasing, method syntax, time truncation vs extraction.",
|
|
323232
|
-
body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n##
|
|
323934
|
+
body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## A Measure Is Already an Aggregate\n\n```malloy\n// WRONG: \"Aggregate expression cannot be aggregate\" - flight_count is a measure\nrun: flights -> { aggregate: busiest is max(flight_count) }\n// RIGHT: aggregate per group, then take the max in a second stage\nrun: flights -> { group_by: carrier, aggregate: n is flight_count } -> { aggregate: busiest is max(n) }\n```\n\nThe message does not name the field. When it appears, look for a model measure you wrapped in another aggregate. If you only need the top row, skip the second stage: `order_by: n desc` with `limit: 1`.\n\n## Window Braces Bind to the Function\n\n```malloy\n// WRONG: \"`partition_by` is not supported for this kind of expression\"\ncalculate: share is sum_cumulative(n) / region_total { partition_by: region, order_by: n desc }\n// RIGHT: the brace goes directly after the window function call\ncalculate: share is sum_cumulative(n) { partition_by: region, order_by: n desc } / region_total\n```\n\nFor a within-group share, compute the denominator in `aggregate:` (`region_total is all(count(), region)`). A window's `{ partition_by }` does not apply to `all()`.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Strings: Apostrophes and Concatenation\n\n```malloy\n// WRONG: \"no viable alternative at input 's'\" - the apostrophe ends the literal\nwhere: name = 'Mac's Diner'\n// RIGHT\nwhere: name = \"Mac's Diner\"\n\n// WRONG: \"unexpected '+'\" / \"no viable alternative at input '||'\"\ngroup_by: route is origin ++ '-' ++ destination\n// RIGHT: there is no concatenation operator\ngroup_by: route is concat(origin, '-', destination)\n```\n\n## A Semicolon Inside a Clause Ends It\n\nBetween clauses a newline, comma or `;` all work. Within one clause, separate fields with commas or newlines: a `;` closes the clause, and the next field is orphaned.\n\n```malloy\n// WRONG: \"no viable alternative at input 'charters'\"\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n```\n\nThe same message appears for an unnamed aggregate after the first: `aggregate: n, max(x)` fails at `max`. Name every entry.\n\n## Compiles, Then Refused by the Warehouse\n\n`Query execution failed: ...` means the Malloy compiled and the warehouse rejected the SQL, often over a type. The position it quotes is in the generated SQL, not in your query. BigQuery, for example, will not partition a window function on a FLOAT64 column. `partition_by:` takes only a field name, so cast in `group_by:` (`season_key is season::string`) and partition on `season_key`, or fix the type in the model."
|
|
323233
323935
|
},
|
|
323234
323936
|
{
|
|
323235
323937
|
name: "malloy-gotchas-rendering",
|
|
@@ -323362,7 +324064,7 @@ view: overview is {
|
|
|
323362
324064
|
{
|
|
323363
324065
|
name: "malloy-html-data-app-runtime",
|
|
323364
324066
|
description: "The JavaScript that drives an in-package HTML data app - calling Publisher.query, building queries from filter state, handling results and errors. For writing the page's data code.",
|
|
323365
|
-
body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a source with `#(authorize)`, a bad given (unknown name or wrong-typed value) fails closed in the authorize check, so it looks like access denied rather than validation. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
|
|
324067
|
+
body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a gated source, a bad given (unknown name or wrong-typed value) fails closed, so it looks like access denied rather than validation. A `#(authorize)` lock also answers 403 for a caller it simply does not admit. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
|
|
323366
324068
|
},
|
|
323367
324069
|
{
|
|
323368
324070
|
name: "malloy-html-data-apps",
|
|
@@ -324006,7 +324708,7 @@ LookML flattens nested field names with double underscores: \`project__id\`, \`l
|
|
|
324006
324708
|
{
|
|
324007
324709
|
name: "malloy-lookml-review/curate-visibility",
|
|
324008
324710
|
description: "LookML Visibility → Malloy Access Modifiers (Step 8). Reference detail for the malloy-lookml-review skill.",
|
|
324009
|
-
body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control).
|
|
324711
|
+
body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control). `#(authorize)` refuses a caller it does not admit with a 403; `#(access_filter)` is the row-scoping sibling. Masking and these gates are independent layers; use the mask for \"everyone sees a coarsened value,\" a gate for \"only some callers see the field at all.\"\n\n## Pattern: Long→wide custom-field pivot\n\nSee `build-derived-tables.md` → \"Long→wide entity-values pivot\" for turning an entity-attribute-value (EAV) custom-field table into wide columns with **filtered aggregates**, the right move when LookML modeled N per-attribute joins.\n\n## Process\n\n1. Read the Visibility Seeds notes captured during discovery\n2. For each field, apply the classification rules above\n3. Present the classified list to the user for confirmation before applying access modifiers"
|
|
324010
324712
|
},
|
|
324011
324713
|
{
|
|
324012
324714
|
name: "malloy-lookml-review/discover",
|
|
@@ -324234,12 +324936,12 @@ Present coverage in this order:
|
|
|
324234
324936
|
{
|
|
324235
324937
|
name: "malloy-materialization",
|
|
324236
324938
|
description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.",
|
|
324237
|
-
body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting an `#(authorize)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **An `#(authorize)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting an `#(authorize)`-gated source**."
|
|
324939
|
+
body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting a `#(access_filter)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **A `#(access_filter)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting a `#(access_filter)`-gated source**."
|
|
324238
324940
|
},
|
|
324239
324941
|
{
|
|
324240
324942
|
name: "malloy-materialization-tuning",
|
|
324241
324943
|
description: "Optimize a package's Malloy Persistence materializations for cost and performance using the malloy-pub CLI and the materialization history. Recommend what to persist, what to stop persisting, and how to schedule/scope it. Use when the user asks to make a package cheaper or faster, tune persistence, decide what to materialize, or review persist/schedule choices.",
|
|
324242
|
-
body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries
|
|
324944
|
+
body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries a `#(access_filter)` gate, check which case it is first. A gate reached only through a `join_*`, or inherited from a base it cannot be attributed to, makes the colocated persist **refuse with a 422** - so do not recommend persisting it at all. Where the gate is provably the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `skill:malloy-materialization`, which also covers where that window does and does not bind).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n ```bash\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n ```bash\n malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n ```bash\n malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg> # 5-field UTC cron\n malloy-pub schedule clear --environment <env> --package <pkg>\n ```\n\n `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
|
|
324243
324945
|
},
|
|
324244
324946
|
{
|
|
324245
324947
|
name: "malloy-model",
|
|
@@ -324461,7 +325163,7 @@ A given is **declared bare** but **referenced with a \`$\` sigil** in expression
|
|
|
324461
325163
|
- **Givens don't auto-inject a \`where:\`.** Unlike \`#(filter)\`, you write the filter expression that references the given yourself (e.g. \`where: dimension ~ $given_name\`).
|
|
324462
325164
|
- **Not every filter maps cleanly.** A filter with no neutral match-all literal default - e.g. a scalar date/number range like \`> @2020-01-01\` - is not a good \`given:\`; keep those on \`#(filter)\`. Two more cases keep using \`#(filter)\`: mandatory scoping filters (\`required\`) and system-injected row-level filters (\`implicit\`), both below.
|
|
324463
325165
|
|
|
324464
|
-
Givens are also the substrate for access control - see "Access Control:
|
|
325166
|
+
Givens are also the substrate for access control - see "Access Control: \`#(authorize)\` and \`#(access_filter)\`" below.
|
|
324465
325167
|
|
|
324466
325168
|
## Legacy: Parameterizable Filters with \`#(filter)\`
|
|
324467
325169
|
|
|
@@ -324532,38 +325234,72 @@ Publisher formats values based on the dimension's data type, \`string\` → \`'v
|
|
|
324532
325234
|
|
|
324533
325235
|
Pass \`bypass_filters=true\` (REST) or \`bypassFilters: true\` (POST body) to skip filter injection entirely. Use sparingly, required-filter governance only works if bypass is restricted to trusted callers.
|
|
324534
325236
|
|
|
324535
|
-
## Access Control:
|
|
325237
|
+
## Access Control: \`#(authorize)\` and \`#(access_filter)\`
|
|
325238
|
+
|
|
325239
|
+
Gate query access to a source over declared \`given:\` values (\`given:\` is Malloy's native runtime-parameter mechanism, the going-forward replacement for \`#(filter)\`). **Two annotations, one question each, and the name tells you which:**
|
|
325240
|
+
|
|
325241
|
+
| Annotation | The question | Body it takes | A denial is |
|
|
325242
|
+
| --- | --- | --- | --- |
|
|
325243
|
+
| \`#(authorize)\` | may this caller reach this source at all? | \`'literal' <op> $GIVEN\`, plus the \`true\`/\`false\` sentinels | **403** |
|
|
325244
|
+
| \`#(access_filter)\` | which rows may they see, once they may? | \`field_path <op> $GIVEN\` | **200**, with their rows |
|
|
324536
325245
|
|
|
324537
|
-
|
|
325246
|
+
Either is an annotation on its own line directly above the \`source:\` line, carrying a **narrow grammar publisher parses itself, not an arbitrary Malloy expression**: one or more terms joined only by \`and\`, with \`<op>\` fixed by the given's declared arity (\`in\` for a list, \`=\` for a scalar).
|
|
325247
|
+
|
|
325248
|
+
**Nothing is inferred from the body.** The annotation you write declares which question you are answering, and a body that does not fit is refused at load naming the other annotation. A source with no annotation of either kind, own or inherited, is unrestricted.
|
|
325249
|
+
|
|
325250
|
+
The lock is DECIDED, before the caller's query compiles: a caller it does not admit gets a 403, never a row count or a \`NULL\` aggregate over data they were refused. The filter is GRAFTED onto the query as a \`where:\`, so a caller it matches nowhere gets a 200 with an empty result. The lock runs first; a caller it refuses never reaches the filter. A 403 also covers either gate failing to apply at all (a field the entry point dropped, a given nobody supplied).
|
|
324538
325251
|
|
|
324539
325252
|
\`\`\`malloy
|
|
324540
325253
|
##! experimental.givens
|
|
324541
325254
|
|
|
324542
325255
|
given:
|
|
324543
325256
|
ROLE :: string
|
|
325257
|
+
ORG_IDS :: string[]
|
|
324544
325258
|
|
|
324545
|
-
#(authorize)
|
|
325259
|
+
#(authorize) 'analyst' = $ROLE
|
|
325260
|
+
#(access_filter) org_id in $ORG_IDS
|
|
324546
325261
|
source: orders is duckdb.table('orders.parquet') extend {
|
|
324547
325262
|
measure: order_count is count()
|
|
324548
325263
|
}
|
|
324549
325264
|
\`\`\`
|
|
324550
325265
|
|
|
324551
|
-
- **
|
|
324552
|
-
- **
|
|
324553
|
-
-
|
|
324554
|
-
- **
|
|
324555
|
-
- **
|
|
324556
|
-
- **
|
|
324557
|
-
|
|
324558
|
-
|
|
324559
|
-
|
|
325266
|
+
- **Nothing outside the two term shapes above parses.** No \`or\`, \`not\`, \`!=\`, \`<\`/\`>\`/\`<=\`/\`>=\`, function calls, bare field/boolean references, or a literal on the right of a row-level term. \`org_id in $GROUPS\` and \`region = $REGION and org_id in $GROUPS\` are legal; \`upper(region) = $REGION\`, \`(org_id in $GROUPS or region = $REGION)\`, and a bare \`#(access_filter) authorized\` are all refused at load with a named cause (see your deployment's reference documentation for the full list).
|
|
325267
|
+
- **Writing a term on the wrong annotation is refused, both ways.** \`#(authorize) org_id in $GROUPS\` is refused naming \`#(access_filter)\`; \`#(access_filter) 'finance' in $GROUPS\` is refused naming \`#(authorize)\`. The second matters: a constant predicate grafted as a row filter would serve a refused caller 200 with zero rows, which is the answer the lock exists to replace.
|
|
325268
|
+
- **A source may declare more than one note on a route: repeats AND together.** \`#(access_filter) region = $REGION\` stacked with a second \`#(access_filter) org_id in $GROUPS\` both apply, and a caller must satisfy every term across every note. \`or\` is still refused wherever it appears, so there is no way to spell "admit if either" inside one gate or across a source's gates; use two extension sources instead, one per admitted population (see the admin pattern below).
|
|
325269
|
+
- **Two exceptions to "every term references a given", and both live on the lock.** \`#(authorize) false\` is a deny-all: that is how you lock a base. \`#(authorize) true\` is an admit-all, and it is what an extension of a locked base needs in order to be open: a source declaring no gate of its own **inherits** its ancestor's, so leaving the annotation off an extension of a \`false\` base inherits the lock rather than lifting it. Neither sentinel is legal on \`#(access_filter)\`. \`false\` may not share a source with another note at all; \`true\` may not share the lock route with one, but is live beside an \`#(access_filter)\` (\`#(authorize) true\` with \`#(access_filter) org_id in $GROUPS\` re-opens a locked base while still scoping the rows). Neither is a term: \`true and org_id in $GROUPS\` is read as an ordinary two-term body whose first term is malformed.
|
|
325270
|
+
- **Inheritance is per route.** Both inherit through \`extend\` the same way, own wins over ancestor on that route only, so a source can be "own" for one and "inherited" for the other. The API reports them separately, each under the field named for its own annotation: \`Source.authorize\` and \`Source.accessFilter\`.
|
|
325271
|
+
- **Generate the admin escape hatch as another extension over the locked base, never as a bypass.** When a model needs an ordinary population plus a wider one, add a second extension whose rule is a lock on the caller:
|
|
325272
|
+
|
|
325273
|
+
\`\`\`malloy
|
|
325274
|
+
given:
|
|
325275
|
+
ORG_IDS :: string[]
|
|
325276
|
+
GROUPS :: string[]
|
|
325277
|
+
|
|
325278
|
+
#(authorize) false
|
|
325279
|
+
source: orders_base is duckdb.table('orders.parquet') extend {}
|
|
325280
|
+
|
|
325281
|
+
#(authorize) true
|
|
325282
|
+
#(access_filter) org_id in $ORG_IDS
|
|
325283
|
+
source: orders is orders_base extend {}
|
|
325284
|
+
|
|
325285
|
+
#(authorize) 'admin' in $GROUPS
|
|
325286
|
+
source: orders_admin is orders_base extend {}
|
|
325287
|
+
\`\`\`
|
|
325288
|
+
|
|
325289
|
+
Each extension replaces the base's \`false\` on the lock route with its own rule. \`orders\` is open to everyone and scoped per row; \`orders_admin\` is locked to the admin group and, carrying no filter, serves every row to whoever passes. This is the shape to generate whenever an author wants a role to see everything: another extension source, not a flag that skips the gate.
|
|
325290
|
+
- **Both only gate from the \`source:\` line.** The same annotation on a \`dimension:\`/\`measure:\`/\`join_*:\`/\`view:\` line, or on a top-level \`query:\`, is refused at load naming the position rather than silently protecting nothing.
|
|
325291
|
+
- **Every given the gate references must be declared on the entry model's own surface, and must carry no default.** A given the model cannot resolve is refused at load. So is a referenced given declared *with* a default: a caller who supplies nothing would get that default and be admitted or excluded by a value the gate's own line never shows, so it is refused rather than reasoned about case by case.
|
|
325292
|
+
- **A scalar/array mismatch between the operator and the given's declared type is a load-time refusal**, not a request-time warehouse error: \`org_id in $GROUPS\` requires \`GROUPS\` to be array-typed, \`region = $REGION\` requires \`REGION\` scalar. Negation is likewise refused outright, so there is no empty-given inversion surprise to warn about.
|
|
325293
|
+
- **Entry point only: not joined, but inherited through \`extend\`.** The gate applies to the source a query enters through. A gate on a source reached only via \`join_*\` **never fires**, at any depth, so anything ungated that joins a locked base hands the base's rows to every caller. A source that \`extend\`s a locked base and declares no gate of its own **does** carry the base's gate; declaring its own annotation replaces it. A source derived from a locked base via a query (\`source: z is locked -> { … }\`) instead **always carries the base's gate in addition to its own**: the derivation recurses into the base unconditionally, so an own gate does not replace it, and the two combine as separate AND'd entries. Pair a locked base with curated extension sources, using access modifiers (\`include { public: …, private: * }\`), so an extension re-exposes only a curated column surface, and keep sensitive sources out of ungated joins.
|
|
325294
|
+
- **A derivation that drops a column the gate reads fails CLOSED.** \`extend { except: org_id }\`, or an \`accept:\` that omits it, leaves the grafted filter unable to compile, so the request is denied rather than served ungated. The one hole to know: dropping the gated column and then \`rename:\`-ing a *different* column onto that exact name grafts successfully and binds the gate to the wrong column. Narrow, but real, so don't recycle a gated column's name. If a derivation's own projection needs to drop the column a row-level term would read, use a source-level term (\`'literal' in/= $GIVEN\`) instead, since it never depends on any projected column.
|
|
325295
|
+
- **The quoted-string and file-level forms are refused at load and no longer exist.** A quoted expression on the \`source:\` line, in either quote, a file-level \`##(…) "<expr>"\` applying to every source in the file, and the earlier \`internal dimension: authorized is <expr>\` form are all retired; the load names the rewrite. Every \`.malloy\` file in a package compiles at load and any failure aborts the package, so a retired-form gate anywhere in the package is refused. Only a declaring file *outside* the package escapes that: it loads and denies every request instead, with no compile-time hint. See your deployment's reference documentation.
|
|
324560
325296
|
- **A gated source can be persisted, but the gating column freezes.** \`storage=\` and \`#@ preaggregate\` refuse a gated source outright; a colocated \`#@ persist\` is admitted when the gate is provably the entry point's own row filter. The gate still runs live on every query, so rows come back filtered - but the column it filters ON is frozen at build time, so a row whose access decision changes keeps being served under the old decision until the next rebuild. Pair \`#@ persist\` on a gated source with a freshness window (\`fallback="live"\`), which is the only control that bounds that - and read \`skill:malloy-materialization\` for where that window binds, because on a standalone Publisher it does not.
|
|
324561
325297
|
|
|
324562
|
-
> **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`.
|
|
325298
|
+
> **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`. Neither annotation is a real boundary unless it sits behind a trusted tier that sets givens from its own verified context, never directly from an untrusted caller. Neither is, on its own, end-user authentication.
|
|
324563
325299
|
>
|
|
324564
|
-
> **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning
|
|
325300
|
+
> **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning these gates into a standalone boundary. Model access on \`given:\` + these two annotations now; it is the surface that carries forward.
|
|
324565
325301
|
|
|
324566
|
-
Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's
|
|
325302
|
+
Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's authorize reference documentation.
|
|
324567
325303
|
|
|
324568
325304
|
## Join Syntax
|
|
324569
325305
|
|
|
@@ -325192,7 +325928,7 @@ source: first_touch is conn.table('events') -> {
|
|
|
325192
325928
|
{
|
|
325193
325929
|
name: "malloy-modeling",
|
|
325194
325930
|
description: "Build semantic models with Malloy for the Malloy Publisher. Read this skill whenever the user asks about modeling data or specifically mentions Malloy.",
|
|
325195
|
-
body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself before writing ANY code, with the tool that matches what you are modelling.\n - Modelling data **already in a package**: `get_context` returns that package's sources, views, and fields (with their docs).\n - Modelling **a database with no package yet**: `get_context` has nothing to return, so use `search_database_schema` instead. It walks the connection's schemas and tables, ranks them against a plain-English description, and gives you each table's columns plus the `source:` line to start from. Take those names verbatim into step 5.\n Never guess field names either way.\n2. **Search docs proactively**: call `search_malloy_docs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **`count()` counts rows; `count(x)` counts distinct values of `x`**: `count(distinct x)` is deprecated, write `count(x)`\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files.\n\n`modeling-notes.md` is an expected output of the workflow, not an optional extra. Start it at step 2 (Propose Scope) and grow it as you work: it persists alongside the model, and its value is as the thing the user argues with at step 3, before source files exist; written after the build it can only document decisions already baked in. Record findings and problems as they are found during discovery (`skill:malloy-discover`), and every unconfirmed decision as an open item. Only when there is no writable workspace do the notes live in the conversation instead.\n\nKeep it compact, with these sections:\n\n```markdown\n# Modeling notes - <package>\n## Scope what was confirmed, what the model is FOR, skip list with reasons\n## Grain and keys proven by query, not by column name\n## Coverage coverage cliffs; columns excluded for nullity\n## Decisions each with its evidence\n## Open decisions ASSUMPTIONS, NOT CONFIRMED: every threshold or definition the user\n has not settled, one entry each, mirrored by a hedge in its #(doc)\n## Validation reconciliation checks performed, and their results\n```\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions. Read each step's skill **before starting that step**, including the decision skills for steps 1–4 (`skill:malloy-discover`, `skill:malloy-scope`, `skill:malloy-define`). They govern what the model says; skipping them to reach the build skills is how unreviewed business logic ships.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags, and any threshold or business convention in it is user-confirmed, distribution-derived, or explicitly flagged in its `#(doc)`** (see `skill:malloy-document` § Mark conventions as conventions). Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent) (user) (user) (user) (agent) (agent) (user) (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art. With no package yet, start from `search_database_schema` to find the tables in the connection |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present the review checklist below; user confirms or corrects |\n| 8. Curate | `skill:malloy-model` | Propose access controls (`explores`, `queryableSources`, access modifiers); always propose, the user decides whether to apply |\n\n### The pauses are the point\n\nThese are governed semantic models: the business decisions in them must be confirmed by a human subject-matter expert, and the **(user)** steps exist to collect that confirmation. They are real stops, not progress reports. A model can be complete, compiling, and fully documented and still be wrong everywhere it guessed; a capable agent can build the whole thing without pausing once, which is exactly the failure mode this workflow exists to prevent.\n\nWhen a decision goes unanswered (the user explicitly declines to decide, or nobody is there to ask), do not silently proceed as if it were settled. Take your best-supported position, label it an assumption in the field's own `#(doc)` (see `skill:malloy-document` § Mark conventions as conventions), record it under \"Open decisions\" in `modeling-notes.md`, and raise it again at Review. An unlabeled assumption is indistinguishable from a confirmed fact, and misleads everyone downstream.\n\n### Step 7 Review is a checklist, not a summary\n\nPresent these to the user, with answers:\n\n- **Which definitions did the user actually confirm?** List them; everything else is an assumption.\n- **Which thresholds and bucket boundaries did you choose?** For each: the evidence (distribution query, metadata, prior art) and the `#(doc)` hedge that marks it.\n- **Which questions were left unanswered?** Each must already carry a labeled assumption and an \"Open decisions\" entry.\n- **Does the headline metric have more than one defensible definition?** If yes, that is a blocking question: put the candidate definitions to the user with their counts side by side, not in a footnote.\n\nThe user confirming this checklist is what makes the model governed. A summary of what you built is not a checkpoint.\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** a data question arrives before any model exists → `skill:malloy-model-as-you-go`. It answers the question with `skill:malloy-analysis`, then codifies what the answer assumed into the model, one question at a time, confirming binding decisions first. The model exists by the end; there is no separate formalize step.\n- **Open-ended exploration** with no intent to keep anything: `skill:malloy-analyze`. If it turns into something worth keeping, formalize via `skill:malloy-model` (`reference/analysis-to-model.md`).\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through Review and propose Curate. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-model-as-you-go`: answer their first real question, codify what it assumed, repeat |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-model-as-you-go`. It answers with `skill:malloy-analysis` and grows the model from what each answer assumed, so there is nothing to formalize afterwards.\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling. No server yet? `skill:malloy-getting-started` covers setup, including the one-command scaffolder (`npm create @malloy-publisher/malloy-package@latest <name>`) and why local authoring needs `--watch-env <env>`: start the server without it and your saved edits are never read.\n\n| Tool | Purpose |\n|------|---------|\n| `get_context` | Ground yourself in a package: its sources, views, and fields |\n| `execute_query` | Run ad-hoc queries for validation |\n| `compile_model` | Compile-check a change and get diagnostics back without running a query |\n| `reload_package` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `search_malloy_docs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n| `search_database_schema` | Find the tables in a database connection by plain-English description, when modelling data that is not in a package yet. Returns each table's columns and the `source:` line to start from. Names and types only: no row value is returned |\n\nNever guess field names. Ground yourself with `get_context` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `compile_model`, picking the scope that matches what you are doing:\n - Adding a new definition or query: the default (`scope: \"append\"`) compiles your text in the model's namespace. Note its diagnostic positions land in the model-plus-your-text concatenation.\n - **Editing an existing definition: `scope: \"file\"`**, with the whole edited file as `source`. It compiles your text AS the file (append would collide with \"Cannot redefine\"), and diagnostics land at the true line numbers of your text.\n - Before saving a change other files import: `scope: \"package\"` with the edited file as `source` runs reload's worker compiler over every `.malloy` and `.malloynb` file against your edit, so a rename that breaks an importer surfaces now instead of at reload. Each diagnostic carries `model`, the file it points at; files hidden from discovery can appear. If `modelPath` does not exactly match an existing file, a warning says the source was treated as new.\n2. **Save** it to the package's model file.\n3. **Reload** with `reload_package`.\n4. **Run** the new view with `execute_query`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback, and a `scope: \"package\"` dry-run with no `source` uses reload's compiler and file selection (imports across files, every `.malloy` and `.malloynb` file as saved) without touching the served model. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `execute_query`. An older Publisher that has `compile_model` but rejects `scope` supports only the append behavior.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ... RIGHT: source: flights is ...\nWRONG: dimension: x as y RIGHT: dimension: y is x\nWRONG: count(*) RIGHT: count()\nWRONG: count(distinct x) RIGHT: count(x)\nWRONG: revenue / order_count RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... } RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns"
|
|
325931
|
+
body: "# STOP - READ BEFORE WRITING ANY MALLOY CODE\n\n> **AI AGENTS: You MUST review this file before writing Malloy code.** Cross-skill references below use logical `skill:` names; load the referenced skill before acting. Before writing code, also read the gotcha skills: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering`.\n\n## Pre-Flight Checklist\n\n1. **Discover first**: ground yourself before writing ANY code, with the tool that matches what you are modelling.\n - Modelling data **already in a package**: `get_context` returns that package's sources, views, and fields (with their docs).\n - Modelling **a database with no package yet**: `get_context` has nothing to return, so use `search_database_schema` instead. It walks the connection's schemas and tables, ranks them against a plain-English description, and gives you each table's columns plus the `source:` line to start from. Take those names verbatim into step 5.\n Never guess field names either way.\n2. **Search docs proactively**: call `search_malloy_docs` BEFORE writing unfamiliar patterns (window functions, query-based sources, pipelines). Don't guess. Malloy syntax is specific and SQL intuition is often wrong.\n3. **Use `skill:malloy-patterns`** to discover available doc topics (YoY, cohorts, rendering, window functions).\n4. **Check diagnostics** after writing: fix the FIRST error first, errors cascade.\n5. **Read the gotcha skills**: `skill:malloy-gotchas-modeling`, `skill:malloy-gotchas-queries`, and `skill:malloy-gotchas-rendering` prevent the most common mistakes.\n\n**Quick syntax reminders:**\n1. **Backtick reserved words:** `` `Date` ``, `` `Hour` ``, `` `Timestamp` ``, `` `Type` ``, `` `number` ``, `` `source` ``\n2. **Use `having:` for aggregate filters**: not `where:` on measures\n3. **Alias joined fields in `group_by`** if using them in `order_by`\n4. **`count()` counts rows; `count(x)` counts distinct values of `x`**: `count(distinct x)` is deprecated, write `count(x)`\n5. **One tag per line**: `# label=\"Revenue\"` and `# currency` on separate lines\n6. **No fixed scale on measures**: use `# currency` not `# currency=usd0m`\n7. **Cast strings for aggregates:** `avg(score::number)` not `avg(score)`\n8. **Boolean columns:** use `= true` not `= 'true'` (no quotes!)\n9. **Read data files in place:** `.csv`, `.parquet`, `.json`, `.ndjson`, and `.xlsx` all work as-is through `duckdb.table('data/file.ext')`. Never convert a file to another format first, and never read one with python or jq to \"have a look\" first: query it. For `.xlsx`, check the row count before trusting it: a workbook with a title row or a blank spacer reads short and reports no error. (Per-format quirks: `skill:malloy-gotchas-modeling`)\n\n## Planning and `modeling-notes.md`\n\nIf the IDE has a native plan mode, use it for the high-level approach: do data exploration during planning, then present a concrete plan for user approval before writing any files.\n\n`modeling-notes.md` is an expected output of the workflow, not an optional extra. Start it at step 2 (Propose Scope) and grow it as you work: it persists alongside the model, and its value is as the thing the user argues with at step 3, before source files exist; written after the build it can only document decisions already baked in. Record findings and problems as they are found during discovery (`skill:malloy-discover`), and every unconfirmed decision as an open item. Only when there is no writable workspace do the notes live in the conversation instead.\n\nKeep it compact, with these sections:\n\n```markdown\n# Modeling notes - <package>\n## Scope what was confirmed, what the model is FOR, skip list with reasons\n## Grain and keys proven by query, not by column name\n## Coverage coverage cliffs; columns excluded for nullity\n## Decisions each with its evidence\n## Open decisions ASSUMPTIONS, NOT CONFIRMED: every threshold or definition the user\n has not settled, one entry each, mirrored by a hedge in its #(doc)\n## Validation reconciliation checks performed, and their results\n```\n\n## 8-Step Modeling Workflow\n\nThe agent orchestrates all steps. Steps marked **(user)** pause for input. Each step has a dedicated skill with full instructions. Read each step's skill **before starting that step**, including the decision skills for steps 1–4 (`skill:malloy-discover`, `skill:malloy-scope`, `skill:malloy-define`). They govern what the model says; skipping them to reach the build skills is how unreviewed business logic ships.\n\n**A field is not complete until it has its definition, `#(doc)` tag, and rendering tags, and any threshold or business convention in it is user-confirmed, distribution-derived, or explicitly flagged in its `#(doc)`** (see `skill:malloy-document` § Mark conventions as conventions). Documentation is part of defining a field, not a separate activity. Read `skill:malloy-document` for full documentation standards (doc string writing, tag ordering).\n\n```\nDISCOVER → SCOPE → SOURCES → DEFINITIONS → BUILD BASE → BUILD JOINED → REVIEW → CURATE\n (silent) (user) (user) (user) (agent) (agent) (user) (user)\n```\n\n| Step | Skill | What Happens |\n|------|-------|-------------|\n| 1. Discover | `skill:malloy-discover` | Read the model and data; scan sources, fields, distributions; detect prior art. With no package yet, start from `search_database_schema` to find the tables in the connection |\n| 2. Propose Scope | `skill:malloy-scope` | Present findings, user selects focus |\n| 3. Propose Sources | `skill:malloy-define` | Propose source plan, user confirms architecture |\n| 4. Propose Definitions | `skill:malloy-define` | Propose fields per base source, user confirms logic |\n| 5. Build Base Sources | `skill:malloy-model` | Write fully documented base source files (one per table), check diagnostics. Read `skill:malloy-document` for doc standards. |\n| 6. Build Joined Sources | `skill:malloy-model` | Write fully documented joined source files, validate. Read `skill:malloy-document` for doc standards. |\n| 7. Review | (none) | Present the review checklist below; user confirms or corrects |\n| 8. Curate | `skill:malloy-model` | Propose the published surface (an `index.malloy` with `export { ... }`) and access controls (access modifiers, gates); always propose, the user decides whether to apply |\n\n### The pauses are the point\n\nThese are governed semantic models: the business decisions in them must be confirmed by a human subject-matter expert, and the **(user)** steps exist to collect that confirmation. They are real stops, not progress reports. A model can be complete, compiling, and fully documented and still be wrong everywhere it guessed; a capable agent can build the whole thing without pausing once, which is exactly the failure mode this workflow exists to prevent.\n\nWhen a decision goes unanswered (the user explicitly declines to decide, or nobody is there to ask), do not silently proceed as if it were settled. Take your best-supported position, label it an assumption in the field's own `#(doc)` (see `skill:malloy-document` § Mark conventions as conventions), record it under \"Open decisions\" in `modeling-notes.md`, and raise it again at Review. An unlabeled assumption is indistinguishable from a confirmed fact, and misleads everyone downstream.\n\n### Step 7 Review is a checklist, not a summary\n\nPresent these to the user, with answers:\n\n- **Which definitions did the user actually confirm?** List them; everything else is an assumption.\n- **Which thresholds and bucket boundaries did you choose?** For each: the evidence (distribution query, metadata, prior art) and the `#(doc)` hedge that marks it.\n- **Which questions were left unanswered?** Each must already carry a labeled assumption and an \"Open decisions\" entry.\n- **Does the headline metric have more than one defensible definition?** If yes, that is a blocking question: put the candidate definitions to the user with their counts side by side, not in a footnote.\n\nThe user confirming this checklist is what makes the model governed. A summary of what you built is not a checkpoint.\n\nPublishing is out of scope for open-source v1. Self-hosters move a finished model into a served package via git and the host's publish path; see `skill:malloy-publish` for the local-to-served handoff.\n\n**Two paths to a model: both produce the same fully documented result:**\n- **Schema-first:** \"Model my data\" → 8-step workflow above using the relevant skills\n- **Analysis-first:** a data question arrives before any model exists → `skill:malloy-model-as-you-go`. It answers the question with `skill:malloy-analysis`, then codifies what the answer assumed into the model, one question at a time, confirming binding decisions first. The model exists by the end; there is no separate formalize step.\n- **Open-ended exploration** with no intent to keep anything: `skill:malloy-analyze`. If it turns into something worth keeping, formalize via `skill:malloy-model` (`reference/analysis-to-model.md`).\n\n## Agent Behavior\n\n**Research before asking.** Present proposals with evidence. Never ask open-ended questions: propose with data and let the user confirm.\n\n**Use business language.** Say \"I simplified the column name\" not \"reserved word replaced.\" Don't expose Malloy internals unless the user asks.\n\n**Describe what you're doing, not which step you're on.** The user doesn't have the skill files open. Say \"I'll propose which tables to include and how they relate\" not \"Steps 3 and 4.\" Say \"Now I'll write the source files\" not \"Moving to Step 5.\" Explain the purpose of each phase in plain language before doing it.\n\n**Present choices as A/B/C.** When asking the user to choose, use lettered options with one-line descriptions. Mark your recommendation.\n\n**Complete all workflow steps.** Once modeling begins, complete through Review and propose Curate. A field without documentation is not finished. If you lose track, re-read the model and your notes. Suggest notebooks at the end.\n\n## Route by Intent\n\n| User says... | Route to |\n|-------------|----------|\n| \"Model my data\", \"create a model\" | 8-step workflow (`skill:malloy-discover`) |\n| \"Model from LookML\" | 8-step with prior art via `skill:malloy-lookml-review` |\n| \"Explore this data\", \"what's interesting?\", \"show me the top X\" | `skill:malloy-analyze` (EDA) |\n| \"Build a dashboard\", \"create views\" on existing model | `skill:malloy-analyze` (views), plus `skill:malloy-charts` or `skill:malloy-notebooks` as needed |\n| \"Build a model but not sure what metrics\" | `skill:malloy-model-as-you-go`: answer their first real question, codify what it assumed, repeat |\n\n**If the user's first message is a data question** (not \"build me a model\"), route to `skill:malloy-model-as-you-go`. It answers with `skill:malloy-analysis` and grows the model from what each answer assumed, so there is nothing to formalize afterwards.\n\n## Additional Support Skills\n\nThese supplemental skills may also be loaded as needed:\n\n- **`skill:malloy`**: Index of Malloy skills and routing guide\n- **`skill:malloy-debug`**: Fix compile errors and interpret diagnostics\n\n## Publisher MCP Tools\n\nEnsure the Publisher MCP tools are configured before modeling. No server yet? `skill:malloy-getting-started` covers setup, including the one-command scaffolder (`npm create @malloy-publisher/malloy-package@latest <name>`) and why local authoring needs `--watch-env <env>`: start the server without it and your saved edits are never read.\n\n| Tool | Purpose |\n|------|---------|\n| `get_context` | Ground yourself in a package: its sources, views, and fields |\n| `execute_query` | Run ad-hoc queries for validation |\n| `compile_model` | Compile-check a change and get diagnostics back without running a query |\n| `reload_package` | Recompile a package from disk so a saved edit becomes queryable by name |\n| `search_malloy_docs` | Search Malloy docs (call BEFORE unfamiliar patterns) |\n| `search_database_schema` | Find the tables in a database connection by plain-English description, when modelling data that is not in a package yet. Returns each table's columns and the `source:` line to start from. Names and types only: no row value is returned |\n\nNever guess field names. Ground yourself with `get_context` to see the sources and fields a package defines.\n\n### The edit-and-run loop\n\nPublisher compiles each configured package at boot and serves that cached model, so a source or view you add afterwards is not queryable by name until you reload the package. The loop is:\n\n1. **Validate** the change with `compile_model`, picking the scope that matches what you are doing:\n - Adding a new definition or query: the default (`scope: \"append\"`) compiles your text in the model's namespace. Note its diagnostic positions land in the model-plus-your-text concatenation.\n - **Editing an existing definition: `scope: \"file\"`**, with the whole edited file as `source`. It compiles your text AS the file (append would collide with \"Cannot redefine\"), and diagnostics land at the true line numbers of your text.\n - Before saving a change other files import: `scope: \"package\"` with the edited file as `source` runs reload's worker compiler over every `.malloy` and `.malloynb` file against your edit, so a rename that breaks an importer surfaces now instead of at reload. Each diagnostic carries `model`, the file it points at; files hidden from discovery can appear. If `modelPath` does not exactly match an existing file, a warning says the source was treated as new.\n2. **Save** it to the package's model file.\n3. **Reload** with `reload_package`.\n4. **Run** the new view with `execute_query`.\n\nA reload that fails to compile is safe: your files are left alone and the previously compiled model keeps serving, with the compile errors returned to you. Compile first anyway for faster feedback, and a `scope: \"package\"` dry-run with no `source` uses reload's compiler and file selection (imports across files, every `.malloy` and `.malloynb` file as saved) without touching the served model. Keep the source of truth outside `publisher_data/`, which is not version-controlled and is wiped by a `--init` restart. If these tools are missing, the Publisher you are connected to predates them; fall back to validating with a throwaway `execute_query`. An older Publisher that has `compile_model` but rejects `scope` supports only the append behavior.\n\n## SQL-to-Malloy Quick Reference\n\n| SQL | Malloy |\n|-----|--------|\n| `COUNT(*)` | `count()` |\n| `COUNT(DISTINCT x)` | `count(x)` |\n| `NOW()` | `now` |\n| `CASE WHEN...END` | `pick...when...else` |\n| `col IN ('a','b')` | `col ? 'a' \\| 'b'` |\n| `COALESCE(a,b)` | `a ?? b` |\n| `CAST(x AS type)` | `x::type` |\n| `DATEDIFF(day, a, b)` | `days(a to b)` |\n| `CONCAT(a, b)` or `a \\|\\| b` | `concat(a, b)` |\n| `TIMESTAMP_DIFF(a, b, SECOND)` | `seconds(b to a)` |\n\n## Critical Rules\n\n1. **All keywords require colons**: `source:`, `dimension:`, `measure:`, `view:`\n2. **Use `is` not `as`**: `dimension: name is expression`\n3. **Arrow operator required**: `run: source -> { operations }`\n4. **Specify join type**: `join_one:`, `join_many:`, `join_cross:`\n5. **Safe division**: `revenue / nullif(count, 0)`\n6. **Group definitions under one keyword**: `measure:` then indent fields beneath\n\n## Common Anti-Patterns\n\n```\nWRONG: source flights is ... RIGHT: source: flights is ...\nWRONG: dimension: x as y RIGHT: dimension: y is x\nWRONG: count(*) RIGHT: count()\nWRONG: count(distinct x) RIGHT: count(x)\nWRONG: revenue / order_count RIGHT: revenue / nullif(order_count, 0)\nWRONG: run: src { ... } RIGHT: run: src -> { ... }\n```\n\n## Reserved Words: Scan Schema First\n\n**Malloy has many reserved words. When in doubt, backtick it.** Most likely to appear as column names:\n\n```\ndate, time, day, month, year, quarter, week, hour, minute, second,\nnumber, string, boolean, type, table, source, index, count, sum, avg, min, max,\ntrue, false, null, is, on, with, all, from, by, in, to, for, select, order_by,\ntop, bottom, desc, asc, row, range, current, window, rank\n```\n\n- `number`: only the bare word needs backticking; `account_number` is fine\n- `source`: reserved; use a different alias like `traffic_source`\n- `string`, `boolean`, `true`, `false`: backtick any column with these exact names\n\n## Gotcha Skills: Read Before Writing Code\n\nThe following skills contain detailed WRONG/RIGHT patterns that prevent the most common Malloy errors. **Read them before writing code:**\n\n- **`skill:malloy-gotchas-modeling`**: Reserved words, NULL checks, date functions, type casts, rename pitfalls, query-based source gotchas, `conn.sql()` anti-pattern\n- **`skill:malloy-gotchas-queries`**: Chart constraints, aggregate filters, joined field aliasing, time truncation vs extraction\n- **`skill:malloy-gotchas-rendering`**: Tag syntax, scale rules, sparkline setup, big_value patterns"
|
|
325196
325932
|
},
|
|
325197
325933
|
{
|
|
325198
325934
|
name: "malloy-notebook-chat",
|
|
@@ -325216,7 +325952,7 @@ Steps to follow when the user asks a question:
|
|
|
325216
325952
|
{
|
|
325217
325953
|
name: "malloy-notebooks",
|
|
325218
325954
|
description: 'Create Malloy notebooks (.malloynb) for interactive dashboards and data stories. Use when user asks to "create a notebook", "make a dashboard notebook", "write a malloynb", "data story", or needs to build reports/visualizations.',
|
|
325219
|
-
body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
|
|
325955
|
+
body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)`/`#(access_filter)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
|
|
325220
325956
|
},
|
|
325221
325957
|
{
|
|
325222
325958
|
name: "malloy-patterns",
|
|
@@ -325226,7 +325962,7 @@ Steps to follow when the user asks a question:
|
|
|
325226
325962
|
{
|
|
325227
325963
|
name: "malloy-phrase-detection",
|
|
325228
325964
|
description: "How to phrase search_text on a get_context call so retrieval returns the fields you need instead of a truncated catalog. Covers target-type classification and decomposition patterns.",
|
|
325229
|
-
body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), not the distinct categorical *values* stored in the data. To find which literal values a categorical dimension holds, target the dimension, then query its distinct values with `execute_query` (see the patterns below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values (no value-search target in v1).** When the user names a literal value like \"premium\" or \"New York City\", target the *dimension* it lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`). Then confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it. The data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value by querying the status dimension's distinct values with `execute_query`.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with `execute_query`.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; enumerate the exact stored values with `execute_query`.\n8. **The quantity asked for is a measure, even when nothing names it.** \"How many titles were released in 2019?\" is asking for a count, but no noun in it is the count: the visible phrases are the subject (\"titles\") and the filter (\"2019\"), and mapping only those yields a `source` and a `dimension` target with no way to answer. Always add a measure target for the quantity itself (`\"the number of titles\"`). This holds for every interrogative that IS the aggregation -- \"how many\", \"how much\", \"how often\" -- and it is the pattern most easily lost when a filter or grouping is the loud part of the sentence. Measured: an answerer got this right on \"How many titles are in the dataset?\" and dropped it on \"How many titles were released in 2019?\", where the year took the attention.\n9. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" do not get their own value targets, they resolve to the city and tier dimensions. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then run `execute_query` to read the city and tier dimensions' distinct values and confirm the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\").\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
|
|
325965
|
+
body: "# Search Target Construction for `get_context`\n\nThe `get_context` tool description defines each field and what a call returns. This skill focuses on the parts you won't get right by default: classifying concepts into target types and splitting ambiguous phrases.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n**Scope of this skill:** the patterns below build `dimension` / `measure` / `view` / `dimensional_value` targets. Phrasing for `source` targets is covered at the end.\n\n**A note on matching:** `get_context` searches over the model (sources, fields, views, and their descriptions), and `dimensional_value` targets search the categorical *values* stored in the data. Some servers don't index values; check the tool's description. On one that doesn't, find a literal value by targeting its dimension, then querying its distinct values with `execute_query` (see \"Where value search isn't available\" below).\n\n## Always send `search_text`\n\n**Do not enumerate.** Omitting `search_text` lists a catalog rather than searching it. Knowing the package narrows *where* to look; it does not substitute for saying *what* you need: if you know the package, that is a reason to scope, not a reason to skip `search_text`. Enumerated listings are capped per source and per entity type, and with no relevance signal the cap drops the fields your question is about while keeping join-path noise.\n\nA bare listing has two legitimate uses. The first is answering \"what data is here?\" when the user has named no subject at all. The second is reading a specific entity you already have the exact name of: scope to its source, set `entity_name`, and pass `search_text: null`, which returns that entity's docstring and Malloy code without spending a search. Every other call carries `search_text` on every target.\n\n## Authoring `search_text` for entity targets\n\nWrite `search_text` as a brief semantic **description** of what you're looking for, not an echo of the user's word. This applies even when you already know the entity name from a prior result: still describe it, don't just repeat the name. That is a rule about how to *phrase* a search, and it does not conflict with the exact-entity lookup above: if you want that one entity's code and docstring rather than a ranked set, pin it with `entity_name` and skip the search entirely. Sending its name back as `search_text` is the move this rule forbids, because it searches for a name instead of either describing a concept or asking for the entity.\n\nOne target per concept is enough: the tool handles phrasing variants internally. Don't pile up dimension targets that point at the same field. Use multiple targets only when they describe genuinely distinct concepts (see \"Non-obvious decomposition patterns\" below).\n\n## Target-type decision guide\n\n- **`dimension`**: categorical attribute to group, filter, or join on. Also used for time and numeric fields.\n - \"region\" becomes `\"the geographic region\"`\n- **`measure`**: aggregation metric (count, sum, average, rate).\n - \"total revenue\" becomes `\"the total revenue or sales amount\"`\n- **`view`**: pre-built analysis. Include one whenever the question sounds like a canned report (summary, breakdown, top-N, trend).\n - \"sales summary\" becomes `\"a summary of sales metrics\"`\n- **`dimensional_value`**: a literal value the user named, stored in some dimension. `search_text` is the value itself, the one target where echoing the user's word is right.\n - \"CyberArk\" stays `\"CyberArk\"`\n - Best practice is to scope the call to the source that holds the value. An unscoped value search covers every indexed dimension in scope, so it is slow on a large package. If you truly don't know the source, you can leave the scope off, but expect a slow call.\n- **`source`**: data domain, for a question that names a subject area rather than fields (phrasing below).\n\n**Resolving categorical values.** When the user names a literal value like \"premium\" or \"New York City\", send a `dimensional_value` target for it, scoped to its source once you know the source. Filter on the exact string it returns: the data may store `\"Premium\"`, `\"PREMIUM\"`, `\"NYC\"`, or `\"New York\"`, and only the data tells you which.\n\n**Where value search isn't available.** Some servers have no value index and return nothing for a `dimensional_value` target, and on others a particular dimension's values may not be indexed. Then target the *dimension* the value lives on (`\"the subscription tier\"`, `\"the city where the subscriber lives\"`), and confirm the exact stored string by querying that dimension's distinct values with `execute_query` before you filter on it.\n\n## Non-obvious decomposition patterns\n\nThese are the rules you won't apply correctly by default:\n\n1. **Adjective + noun, split.** \"active users\" becomes two dimension targets: one for the attribute (`\"the status of the user account\"`) and one for the noun (`\"the user or account holder\"`). Resolve the modifier (\"active\") to the exact stored value with a `dimensional_value` target, or the status dimension's distinct values where values aren't indexed.\n2. **Ambiguous concept, cover both types.** \"rating\", \"duration\", and the like could be either a dimension or a measure: create one target of each type.\n3. **Time references are dimensions.** \"last year\" becomes a dimension target for the relevant date field (`\"the date the event occurred\"`).\n4. **Numeric ranges are dimensions.** \"aged 50\", \"revenue over $1M\" become dimension targets; the comparison is applied in the query, not matched as text.\n5. **Categorical strings that look numeric are still dimensions.** \"18-30\", \"<5 days\", \"tier 2\" are stored as literal strings on a dimension. Target that dimension, then confirm the exact string with a `dimensional_value` target, or `execute_query` where values aren't indexed.\n6. **\"Top N\" without a named measure, add a ranking measure.** \"top 6 products\" becomes a measure for the ranking concept (`\"the performance metric for a product\"`) plus a dimension for the entity. If the measure is explicit (\"top products by total sales\"), use it directly and skip the generic ranking measure.\n7. **Multiple values for one concept, one dimension target.** Several values (\"premium and basic\") still map to a single dimension target for the parent field; confirm the exact stored values with `dimensional_value` targets, or `execute_query` where values aren't indexed.\n8. **The quantity asked for is a measure, even when nothing names it.** \"How many titles were released in 2019?\" is asking for a count, but no noun in it is the count: the visible phrases are the subject (\"titles\") and the filter (\"2019\"), and mapping only those yields a `source` and a `dimension` target with no way to answer. Always add a measure target for the quantity itself (`\"the number of titles\"`). This holds for every interrogative that IS the aggregation -- \"how many\", \"how much\", \"how often\" -- and it is the pattern most easily lost when a filter or grouping is the loud part of the sentence. Measured: an answerer got this right on \"How many titles are in the dataset?\" and dropped it on \"How many titles were released in 2019?\", where the year took the attention.\n9. **A population qualifier is a target, and so is the one the question omits.** Words like \"real\", \"actual\", \"genuine\", \"live\" or \"production\" are not filler: they name rows the model marks for exclusion. Target the flag itself (`\"the flag marking synthetic, test or monitoring traffic\"`), not just the noun they modify. Add one such target even when the question carries no qualifier at all, because a table of events, requests, sessions or logs usually holds test, internal or cancelled rows and nothing in the wording will say so. Resolve it to the model's own flag rather than inventing a filter on an id or a name; a hand-rolled exclusion and the documented one rarely select the same rows.\n\n## Worked example\n\n**User:** \"Customer churn in NYC over the last year for premium and basic subscribers\"\n\nThe targets for this question:\n\n| target_type | search_text |\n|---|---|\n| `measure` | `\"the rate at which customers leave the service\"` |\n| `dimension` | `\"the city where the subscriber lives\"` |\n| `dimension` | `\"the date the subscription was canceled\"` |\n| `dimension` | `\"the tier of the subscription\"` |\n| `view` | `\"subscriber churn or retention analysis\"` |\n\nKey moves: time (\"last year\") becomes a dimension on the cancellation date; \"NYC\" and \"premium/basic subscribers\" resolve to the city and tier dimensions in this first call, because the source is not known yet and an unscoped value search is slow. One `view` target is included to surface any canned churn analysis.\n\nThe response returns the source these fields live on (here `subscriptions`) with the matched fields on its card. Then send a second call scoped to `subscriptions`, with `dimensional_value` targets for `\"NYC\"`, `\"premium\"` and `\"basic\"`, to get the exact strings to filter on (\"New York City\" vs \"NYC\", \"premium\" vs \"Premium\"). Where values aren't indexed, run `execute_query` on the city and tier dimensions' distinct values instead.\n\n## Authoring `search_text` for `source` targets\n\nAim for **3-8 words that name the entity and its business process**. Don't include filter values, time ranges, or aggregations: those belong in entity targets.\n\n| Too vague | Over-specific (entity-shaped) | Good |\n|---|---|---|\n| `\"orders\"` | `\"total order revenue by customer last year\"` | `\"customer order history and line items\"` |\n| `\"customer data\"` | `\"premium subscribers who churned in NYC\"` | `\"subscriber accounts and churn\"` |\n| `\"metrics\"` | `\"monthly revenue variance by account\"` | `\"sales pipeline and revenue forecasts\"` |\n\nHeuristics:\n\n1. **Translate, don't echo.** \"How did sales go last month?\" becomes `\"sales order revenue\"`, not `\"sales last month\"`.\n2. **Differentiate by data shape or business process, not by the user's industry, brand, or product category.** Prefer `\"order fulfillment and shipping\"` over `\"ecommerce data\"` when multiple commerce-ish packages exist. Do NOT add words like `\"eyewear\"`, `\"subscription box\"`, `\"Acme Corp\"`, or the user's specific vertical/brand. Source summaries describe data structure, not the customer's vertical, so those words add noise and can hurt matching.\n3. **Retry with alternative phrasings** if the right source isn't in the results before concluding it's missing."
|
|
325230
325966
|
},
|
|
325231
325967
|
{
|
|
325232
325968
|
name: "malloy-publish",
|
|
@@ -325279,28 +326015,47 @@ If it doesn't exist, create one. Suggest a package name based on the model conte
|
|
|
325279
326015
|
|
|
325280
326016
|
### Curating discovery & the query boundary (optional)
|
|
325281
326017
|
|
|
325282
|
-
|
|
326018
|
+
A package with no \`index.malloy\` and no \`explores\` exposes **everything**: every model is listed and every source is directly queryable.
|
|
326019
|
+
|
|
326020
|
+
To curate, add an **\`index.malloy\`** at the package root. Publisher reads it as the package's published surface, so no manifest field is involved:
|
|
326021
|
+
|
|
326022
|
+
\`\`\`malloy
|
|
326023
|
+
// index.malloy
|
|
326024
|
+
import "order_analysis.malloy"
|
|
326025
|
+
import "staging.malloy"
|
|
326026
|
+
|
|
326027
|
+
export { orders, customers }
|
|
326028
|
+
\`\`\`
|
|
326029
|
+
|
|
326030
|
+
What it exports is what agents discover **and** what may be queried. Everything else still compiles, and other models can import, join and extend it, but a direct query against it is refused with a 404 - indistinguishable from a source that does not exist. Reach for this when you have raw/staging/scaffolding sources that exist to build a curated entry point and you don't want agents landing on, or querying, them directly.
|
|
326031
|
+
|
|
326032
|
+
**Address queries to the surface.** Once a package has an \`index.malloy\`, \`.../models/staging.malloy/query\` is no longer a query entry point, *even for a source that file declares itself*. Use \`.../models/index.malloy/query\`. If you are debugging a refusal rather than authoring, \`skill:malloy-source-unreachable\` covers the three ways a source can be out of reach and how to tell them apart.
|
|
326033
|
+
|
|
326034
|
+
**A surface can be layered.** An \`index.malloy\` may front a file that fronts another. A source re-exported through a chain of files stays queryable through the surface at any depth, because admission follows the declaration rather than the path taken to it.
|
|
326035
|
+
|
|
326036
|
+
**Curation hides a landing point, not a column.** A published source may \`join\` an unpublished one, and a query grouping by a joined field returns that field's values normally. If a column must not be readable, do not join it into something you publish; gate it with \`#(authorize)\` instead.
|
|
325283
326037
|
|
|
325284
|
-
|
|
326038
|
+
**Leaving a source out does not put it out of reach, but there is a condition.** \`export { ... }\` also decides what an *importing* file may see, which is Malloy's rule rather than Publisher's. A file that declares no \`export\` hands an importer everything it declares, so an unpublished source stays importable and joinable from the file that declares it. A file that *does* declare one hands over exactly that list: importing a file whose \`export\` omits a source and then naming it fails to compile with \`Reference to undefined object\`. Put an \`export\` on a mid-layer file only when you mean to narrow what its importers can build on, not just what Publisher lists.
|
|
326039
|
+
|
|
326040
|
+
**About \`export { … }\`:** the surface filters which *files* are listed; \`export { … }\` (a Malloy statement) filters which *sources within a file* are exposed, and the two compose. You usually don't write it in a leaf model: a file with **no** \`export\` exposes all of its own top-level sources. It must appear after the definitions it names. See [Malloy: Imports & Exports](https://docs.malloydata.dev/documentation/language/imports).
|
|
326041
|
+
|
|
326042
|
+
### The older manifest fields
|
|
326043
|
+
|
|
326044
|
+
Both still work and are not going away in this release. Both are deprecated where \`index.malloy\` replaces them, and a package using them that way gets a load-time warning naming the replacement. The two uses it cannot replace, an \`explores\` naming several files and \`queryableSources: "all"\`, stay supported without a warning. Do not add either key to a new package that does not need one of those two.
|
|
325285
326045
|
|
|
325286
326046
|
\`\`\`json
|
|
325287
326047
|
{
|
|
325288
326048
|
"name": "ecommerce",
|
|
325289
|
-
"version": "0.0.1",
|
|
325290
|
-
"description": "Orders, customers, and revenue analysis",
|
|
325291
326049
|
"explores": ["order_analysis.malloy", "customer_health.malloy"],
|
|
325292
|
-
"queryableSources": "
|
|
326050
|
+
"queryableSources": "all"
|
|
325293
326051
|
}
|
|
325294
326052
|
\`\`\`
|
|
325295
326053
|
|
|
325296
|
-
- **\`explores\`** (\`string[]\`) -
|
|
325297
|
-
- **\`queryableSources\`** (\`"declared"\` | \`"all"\`, default \`"declared"\`) - the query boundary.
|
|
325298
|
-
|
|
325299
|
-
**About \`export { … }\`:** \`explores\` filters which *files* are listed; \`export { … }\` (a Malloy statement) filters which *sources within a file* are exposed, the two compose. You usually don't write it: a file with **no** \`export\` exposes all of its own top-level sources. Add \`export { orders, customers }\` to a file to expose only those and keep imported/scaffolding helpers out of discovery (it must appear after the definitions it names). See [Malloy: Imports & Exports](https://docs.malloydata.dev/documentation/language/imports).
|
|
326054
|
+
- **\`explores\`** (\`string[]\`) - model file paths relative to the package root, naming the surface. Reach for it for the one thing \`index.malloy\` cannot express: a surface spanning **several** files. An explicit \`explores\` always wins over the convention, and a package with both an \`index.malloy\` and an \`explores\` that omits it carries a warning saying so. An entry that doesn't resolve to a real \`.malloy\` file surfaces in \`exploresWarnings\`; publishing a package that has any is rejected, so fix the path before publishing. An explicit \`"explores": []\` means "do not curate" and suppresses the convention.
|
|
326055
|
+
- **\`queryableSources\`** (\`"declared"\` | \`"all"\`, default \`"declared"\`) - the query boundary. \`"declared"\` is already the default, so the key changes nothing. **\`"all"\` is the exception, and \`index.malloy\` does not replace it:** it curates listings while leaving every source queryable by name, and a surface derived from an \`index.malloy\` always enforces the boundary. If you want listings-only curation, keep both keys.
|
|
325300
326056
|
|
|
325301
|
-
**Why curate here:** declaring \`explores\` routes agents to the well-documented curated sources instead of raw tables, and \`queryableSources: "declared"\` keeps them from reaching the hidden sources by name. The two axes compose: list a file in \`explores\` for its models to be discoverable, and \`export\` a source within that file for it to be a landing point.
|
|
325302
326057
|
|
|
325303
|
-
> **Not access control.**
|
|
326058
|
+
> **Not access control.** The surface gates the query surface (the query endpoints, REST and MCP alike), not compile and not raw file retrieval by exact path: \`/compile\` and \`compile_model\` are deliberately exempt, because compile is the authoring loop and the boundary is discovery curation. It doesn't restrict *who* may query, only *what* is queryable by name. Queryable sources are the union of every listed file's \`export {}\` closure, whichever listed model path a query addresses them through. To gate access by caller-supplied identity/role, use \`#(authorize)\` on the source (and \`#(access_filter)\` to scope rows), see \`skill:malloy-model\` § Access Control and \`docs/authorize.md\`. Discovery curation and these gates are independent layers.
|
|
325304
326059
|
|
|
325305
326060
|
The manifest also carries a \`scope\` field (\`"package"\` | \`"version"\`, default \`"package"\`) controlling whether persisted/materialized artifacts are shared across published versions or owned by a single version, and a \`materialization\` field configuring that persistence policy (a cron \`schedule\` or a \`freshness\` window). Both are unrelated to discovery curation; there is no per-source \`sharing\` or \`schedule\` field, that was retired in favor of the single package-level \`scope\` and \`materialization\`.
|
|
325306
326061
|
|
|
@@ -325433,10 +326188,10 @@ Wrong: \`pick('Premium') { ... }\` (that's not Malloy syntax).
|
|
|
325433
326188
|
**Window functions with \`calculate:\`:** running totals, \`lag()\`, \`lead()\`, and other window operations belong in \`calculate:\`, not \`aggregate:\`.
|
|
325434
326189
|
\`\`\`malloy
|
|
325435
326190
|
run: source -> {
|
|
325436
|
-
group_by:
|
|
326191
|
+
group_by: order_month is order_date.month
|
|
325437
326192
|
aggregate: revenue
|
|
325438
326193
|
calculate: prev_month_revenue is lag(revenue)
|
|
325439
|
-
order_by:
|
|
326194
|
+
order_by: order_month
|
|
325440
326195
|
}
|
|
325441
326196
|
\`\`\`
|
|
325442
326197
|
|
|
@@ -325474,15 +326229,15 @@ where: order_date >= @2020-01-01
|
|
|
325474
326229
|
|
|
325475
326230
|
### Truncation accessors return timestamps, not integers
|
|
325476
326231
|
|
|
325477
|
-
\`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer
|
|
326232
|
+
\`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer.
|
|
325478
326233
|
|
|
325479
|
-
Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`.
|
|
326234
|
+
Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`.
|
|
325480
326235
|
|
|
325481
|
-
### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved
|
|
326236
|
+
### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved as names
|
|
325482
326237
|
|
|
325483
|
-
|
|
326238
|
+
They cannot name an output field. The functions are fine: \`month(order_date)\` extracts 1-12 anywhere, including \`where:\`.
|
|
325484
326239
|
|
|
325485
|
-
Wrong: \`group_by: month is order_date.month\` →
|
|
326240
|
+
Wrong: \`group_by: month is order_date.month\` → \`'month' is a reserved word, so to use it as a name you must quote it\`
|
|
325486
326241
|
Right: \`group_by: order_month is order_date.month\`
|
|
325487
326242
|
Right (when a column is literally named \`month\`): \`\` group_by: \`month\` \`\`
|
|
325488
326243
|
|
|
@@ -325598,7 +326353,7 @@ run: source -> {
|
|
|
325598
326353
|
}
|
|
325599
326354
|
\`\`\`
|
|
325600
326355
|
|
|
325601
|
-
Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A
|
|
326356
|
+
Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A chart annotation left as the last line inside \`{ }\` fails with *"Parser enountered unexpected statement type 'unimplemented'"* (the compiler's spelling); one after the closing \`}\` fails with *"Object annotation not connected to any object"*.
|
|
325602
326357
|
|
|
325603
326358
|
Read the \`malloy-charts\` skill for chart types, properties, data shape requirements, and selection guidance.
|
|
325604
326359
|
|
|
@@ -325609,11 +326364,10 @@ Read the error against the tables above and below. Most failures match a known p
|
|
|
325609
326364
|
| Error message | Likely cause / fix |
|
|
325610
326365
|
|---|---|
|
|
325611
326366
|
| \`Cannot compare a timestamp to a number\` | Comparing \`date.year\` to an integer. Use \`date >= @2020-01-01\` instead. |
|
|
325612
|
-
| \`no viable alternative at input '<word>'\` |
|
|
326367
|
+
| \`no viable alternative at input '<word>'\` | Often a \`;\` between fields within one clause - fields under one \`aggregate:\`/\`group_by:\` are comma- or newline-separated (the error points at the field right after the \`;\`). |
|
|
325613
326368
|
| \`'<field_name>' is not defined\` | Field doesn't exist in the source. Re-check against the model definition; you may have stripped a join prefix. |
|
|
325614
|
-
| \`missing {DAY, HOUR, MINUTE, MONTH, QUARTER, SECOND, WEEK, YEAR}\` | Chained date property too deep (e.g., \`.month.something\`). Stop at the first truncation. |
|
|
325615
326369
|
| \`field is a bar chart, but is not a repeated record\` | Chart annotation placed inside \`{ }\`. Move \`# bar_chart\` above \`run:\` / \`view:\` / \`nest:\`. |
|
|
325616
|
-
| \`Parser
|
|
326370
|
+
| \`Parser enountered unexpected statement\` | Spelled that way by the compiler. Most often a chart annotation left as the last line inside \`{ }\` - move it above \`run:\`. Also syntax Malloy doesn't allow in that position (e.g., \`pick\` inside a nested view). |
|
|
325617
326371
|
| Query silently returns zero rows | Filter value mismatch (case, spelling, format). Run a distinct-values query on the dimension to confirm the literal. |
|
|
325618
326372
|
|
|
325619
326373
|
## Syntax Help
|
|
@@ -326304,6 +327058,74 @@ Then hand off to modeling: use your modeling workflow to turn the confirmed scop
|
|
|
326304
327058
|
## Done
|
|
326305
327059
|
|
|
326306
327060
|
Scope confirmed, in the conversation and in \`modeling-notes.md\`: tables in scope, analytical focus, model kind and what it rules out, and what's deferred. Continue with your modeling workflow.`
|
|
327061
|
+
},
|
|
327062
|
+
{
|
|
327063
|
+
name: "malloy-source-unreachable",
|
|
327064
|
+
description: `Work out why a source is missing from get_context, or why a query against it was refused with a 404 or a 403, and what to do next. Use when a source you expect is absent from discovery, when execute_query answers "No queryable source", when a query that used to work stops working, or when deciding whether a package's curation is hiding something you need.`,
|
|
327065
|
+
body: `# A source you cannot reach
|
|
327066
|
+
|
|
327067
|
+
> **Tool names** are written bare here - \`get_context\`, \`execute_query\`, \`get_status\`, \`compile_model\`. The exact prefixed name depends on the host surface; match each against the tools you actually have.
|
|
327068
|
+
|
|
327069
|
+
Most packages publish a curated surface, so a source being absent or refused is usually the package working as its author intended, not a fault. Read the refusal before working around it, because the three things that can refuse you need three different responses and they are told apart by the status code.
|
|
327070
|
+
|
|
327071
|
+
## The three answers
|
|
327072
|
+
|
|
327073
|
+
| what you got | what it means | what to do |
|
|
327074
|
+
| --- | --- | --- |
|
|
327075
|
+
| **404**, "No queryable source", "No queryable model" or "Query target is not queryable" | the source is not on the package's published surface, or does not exist. The two are deliberately indistinguishable so that a refusal cannot be used to probe for hidden names | query something on the surface, or ask the package's author to publish it |
|
|
327076
|
+
| **403**, "Access denied" | the source exists and is on the surface, but an \`#(authorize)\` gate did not admit you | supply the givens the gate reads, or accept that this caller may not read it |
|
|
327077
|
+
| **200** with zero rows | you were admitted, and an \`#(access_filter)\` narrowed the rows to none of them | this is a real answer. Report it as "no matching rows", never as an error |
|
|
327078
|
+
|
|
327079
|
+
A 403 names a source, so it tells you the source exists. A 404 tells you nothing at all. That asymmetry is deliberate: a source that is both hidden AND gated answers **404**, so you can never use a 403 to discover that a hidden name is real.
|
|
327080
|
+
|
|
327081
|
+
## Addressing the surface, which is the most common 404
|
|
327082
|
+
|
|
327083
|
+
A query names a model file. When a package curates its surface, only files on that surface are valid entry points, **even for a source that file declares itself**:
|
|
327084
|
+
|
|
327085
|
+
\`\`\`
|
|
327086
|
+
POST .../packages/sales/models/index.malloy/query {"sourceName": "orders"} -> 200
|
|
327087
|
+
POST .../packages/sales/models/orders.malloy/query {"sourceName": "orders"} -> 404
|
|
327088
|
+
\`\`\`
|
|
327089
|
+
|
|
327090
|
+
Same source, same package. \`orders.malloy\` is off the surface, so it is not an entry point.
|
|
327091
|
+
|
|
327092
|
+
**Use the \`model_path\` that \`get_context\` gave you, verbatim.** Its \`source_info.resource_id\` holds \`environment\`, \`package\`, \`model_path\` and \`source\`, and those are exactly \`execute_query\`'s \`environmentName\`, \`packageName\`, \`modelPath\` and \`sourceName\`. On a curated package the \`model_path\` is the surface file, not the file that declares the source. Substituting the declaring file because it looks more correct is how this 404 happens.
|
|
327093
|
+
|
|
327094
|
+
## When the source is not in \`get_context\` at all
|
|
327095
|
+
|
|
327096
|
+
Work down this list. The first three are far more common than the last.
|
|
327097
|
+
|
|
327098
|
+
1. **It is off the surface.** The package publishes a curated set and this source is not in it. Nothing you can do from the query side; it is a package edit.
|
|
327099
|
+
2. **The package failed to load, or is serving a stale model.** Call \`get_status\`. A package that never loaded is absent from listings entirely, which reads exactly like "does not exist". A package whose last reload failed to compile is listed and answering, from the model it compiled *before* that save, and carries \`stale: true\`. Neither is visible from a listing alone.
|
|
327100
|
+
3. **You are talking to a different server than you think.** Call \`list_packages\` and check the environment and package names are the ones you expect. A stale \`.mcp.json\` outlives the server that wrote it, and another Publisher may hold that port.
|
|
327101
|
+
4. **Your search phrasing missed it.** Retry with a bare target (no \`search_text\`) to enumerate rather than rank. If it appears there, the source exists and was a ranking miss.
|
|
327102
|
+
|
|
327103
|
+
## What curation does not do
|
|
327104
|
+
|
|
327105
|
+
- **It does not hide fields.** A published source may \`join\` an unpublished one, and a query grouping by a joined field returns that field's values normally. Hiding a source removes it as a landing point; it does not redact columns a published source pulls in.
|
|
327106
|
+
- **It does not gate \`/compile\`.** \`compile_model\` is exempt, because compile is the authoring loop. A hidden source can still be compile-checked, and that is intended. The exception is a hidden source that is also gated, which answers 404 at compile too.
|
|
327107
|
+
- **It is not access control.** Curation answers "what is queryable by name", not "who may query it". Identity is \`#(authorize)\` and \`#(access_filter)\`. A source is not protected by being hidden.
|
|
327108
|
+
|
|
327109
|
+
## Reaching an unpublished source from your own model
|
|
327110
|
+
|
|
327111
|
+
If you are writing Malloy rather than just querying, an unpublished source is still usable, with one condition that trips people.
|
|
327112
|
+
|
|
327113
|
+
\`export { ... }\` decides what an **importing file** can see, which is Malloy's rule and not Publisher's:
|
|
327114
|
+
|
|
327115
|
+
- a file that declares **no** \`export\` hands an importer everything it declares, so import the file that declares the source and join it normally;
|
|
327116
|
+
- a file that **does** declare one hands over exactly that list. Importing a file whose \`export\` omits the source and then naming it fails to compile with \`Reference to undefined object\`.
|
|
327117
|
+
|
|
327118
|
+
So an unpublished source is reachable through a file that exports it, or that exports nothing, and not through a file whose \`export\` leaves it out. Re-export chains are fine at any depth: a source re-exported through several files stays queryable through the surface, because admission follows the declaration rather than the path taken to it.
|
|
327119
|
+
|
|
327120
|
+
## Before you report a source as missing
|
|
327121
|
+
|
|
327122
|
+
Confirm all three, in this order. Each one has been mistaken for "the data is not there".
|
|
327123
|
+
|
|
327124
|
+
1. \`get_status\` shows the package loaded and not \`stale\`.
|
|
327125
|
+
2. \`list_packages\` names the environment and package you meant.
|
|
327126
|
+
3. An enumerating \`get_context\` call (a target with no \`search_text\`) does not list it.
|
|
327127
|
+
|
|
327128
|
+
Only then is it genuinely not published, and the answer to the user is that the package does not expose it, not that the data does not exist.`
|
|
326307
327129
|
}
|
|
326308
327130
|
]
|
|
326309
327131
|
};
|
|
@@ -327699,7 +328521,7 @@ async function withBigQueryLookupRetry(work, parentJobId) {
|
|
|
327699
328521
|
parentJobId,
|
|
327700
328522
|
error: error instanceof Error ? error.message : String(error)
|
|
327701
328523
|
});
|
|
327702
|
-
await new Promise((
|
|
328524
|
+
await new Promise((resolve7) => setTimeout(resolve7, BIGQUERY_LOOKUP_BACKOFF_MS * attempt));
|
|
327703
328525
|
}
|
|
327704
328526
|
}
|
|
327705
328527
|
}
|
|
@@ -330384,11 +331206,11 @@ var promisifyStore = (passedStore) => {
|
|
|
330384
331206
|
|
|
330385
331207
|
class PromisifiedStore {
|
|
330386
331208
|
async increment(key) {
|
|
330387
|
-
return new Promise((
|
|
331209
|
+
return new Promise((resolve7, reject2) => {
|
|
330388
331210
|
legacyStore.incr(key, (error, totalHits, resetTime) => {
|
|
330389
331211
|
if (error)
|
|
330390
331212
|
reject2(error);
|
|
330391
|
-
|
|
331213
|
+
resolve7({ totalHits, resetTime });
|
|
330392
331214
|
});
|
|
330393
331215
|
});
|
|
330394
331216
|
}
|
|
@@ -330518,9 +331340,9 @@ var rateLimit = (passedOptions) => {
|
|
|
330518
331340
|
}
|
|
330519
331341
|
}
|
|
330520
331342
|
const middleware = handleAsyncErrors(async (request, response, next) => {
|
|
330521
|
-
const closePromise = config.skipFailedRequests && new Promise((
|
|
330522
|
-
const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((
|
|
330523
|
-
const errorPromise = config.skipFailedRequests && new Promise((
|
|
331343
|
+
const closePromise = config.skipFailedRequests && new Promise((resolve7) => response.once("close", resolve7));
|
|
331344
|
+
const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((resolve7) => response.once("finish", resolve7));
|
|
331345
|
+
const errorPromise = config.skipFailedRequests && new Promise((resolve7) => response.once("error", resolve7));
|
|
330524
331346
|
debug2("requested %o", request.originalUrl);
|
|
330525
331347
|
debug2("request from ip %o", request.ip);
|
|
330526
331348
|
const skip = await config.skip(request, response);
|
|
@@ -331862,8 +332684,8 @@ mainServer.timeout = 600000;
|
|
|
331862
332684
|
mainServer.keepAliveTimeout = 600000;
|
|
331863
332685
|
mainServer.headersTimeout = 600000;
|
|
331864
332686
|
var resolveRestBound = () => {};
|
|
331865
|
-
var restBound = new Promise((
|
|
331866
|
-
resolveRestBound =
|
|
332687
|
+
var restBound = new Promise((resolve8) => {
|
|
332688
|
+
resolveRestBound = resolve8;
|
|
331867
332689
|
});
|
|
331868
332690
|
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
331869
332691
|
resolveRestBound();
|