@malloy-publisher/server 0.5.1 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +71 -37
- package/dist/app/assets/{EnvironmentPage-B02geyxh.js → EnvironmentPage-C4IrQrKG.js} +1 -1
- package/dist/app/assets/{HomePage-BdKzZEto.js → HomePage-DDyGPzmK.js} +1 -1
- package/dist/app/assets/{LightMode-BRAs1k98.js → LightMode-BpHRmI5M.js} +1 -1
- package/dist/app/assets/{MainPage-D9XZbGvk.js → MainPage-BHleF-Kb.js} +1 -1
- package/dist/app/assets/{ModelPage-B-U47DzF.js → ModelPage-Cgw-SbZi.js} +2 -2
- package/dist/app/assets/{PackagePage-CDp9A3_k.js → PackagePage-Cy0c37D0.js} +1 -1
- package/dist/app/assets/{RenderedResult-CTD2Ej2S.es-BZsZAiwo.js → RenderedResult-DSgqZUiN.es-CL0FEank.js} +2 -2
- package/dist/app/assets/{RouteError-CSsLG99c.js → RouteError-DBMc9ccU.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-C4u5l49i.js → ThemeEditorPage-ByJlsmNu.js} +1 -1
- package/dist/app/assets/{ToggleButtonGroup-IBhUYkv_.js → ToggleButtonGroup-BHEVw4Ti.js} +1 -1
- package/dist/app/assets/{index-CshLRozl.js → index-BdwkgpbO.js} +1 -1
- package/dist/app/assets/{index-Bd7WYNw_.js → index-C5BtJ1GX.js} +4 -4
- package/dist/app/assets/{index-BKNRDF5y.js → index-Dyxfz1yO.js} +1 -1
- package/dist/app/assets/{index-DOXSuS-a.js → index-c3i6ScGn.js} +1 -1
- package/dist/app/assets/{index--DAcpR5Y.js → index-rWWmF7hn.js} +1 -1
- package/dist/app/assets/{index.es-Bj77sE9l.js → index.es-VI5lesX5.js} +4 -4
- package/dist/app/index.html +1 -1
- package/dist/cpufeatures-1yrn0vtw.node +0 -0
- package/dist/package_load_worker.mjs +693 -352
- package/dist/server.mjs +1068 -574
- package/package.json +1 -1
package/dist/server.mjs
CHANGED
|
@@ -232535,12 +232535,23 @@ var require_bcrypt_pbkdf = __commonJS((exports, module) => {
|
|
|
232535
232535
|
};
|
|
232536
232536
|
});
|
|
232537
232537
|
|
|
232538
|
+
// ../../node_modules/cpu-features/build/Release/cpufeatures.node
|
|
232539
|
+
var require_cpufeatures = __commonJS((exports, module) => {
|
|
232540
|
+
module.exports = __require("./cpufeatures-1yrn0vtw.node");
|
|
232541
|
+
});
|
|
232542
|
+
|
|
232543
|
+
// ../../node_modules/cpu-features/lib/index.js
|
|
232544
|
+
var require_lib8 = __commonJS((exports, module) => {
|
|
232545
|
+
var binding = require_cpufeatures();
|
|
232546
|
+
module.exports = binding.getCPUInfo;
|
|
232547
|
+
});
|
|
232548
|
+
|
|
232538
232549
|
// ../../node_modules/ssh2/lib/protocol/constants.js
|
|
232539
232550
|
var require_constants42 = __commonJS((exports, module) => {
|
|
232540
232551
|
var crypto3 = __require("crypto");
|
|
232541
232552
|
var cpuInfo;
|
|
232542
232553
|
try {
|
|
232543
|
-
cpuInfo = (
|
|
232554
|
+
cpuInfo = require_lib8()();
|
|
232544
232555
|
} catch {}
|
|
232545
232556
|
var { bindingAvailable, CIPHER_INFO, MAC_INFO } = require_crypto();
|
|
232546
232557
|
var eddsaSupported = (() => {
|
|
@@ -247183,7 +247194,7 @@ var require_keygen = __commonJS((exports, module) => {
|
|
|
247183
247194
|
});
|
|
247184
247195
|
|
|
247185
247196
|
// ../../node_modules/ssh2/lib/index.js
|
|
247186
|
-
var
|
|
247197
|
+
var require_lib9 = __commonJS((exports, module) => {
|
|
247187
247198
|
var {
|
|
247188
247199
|
AgentProtocol,
|
|
247189
247200
|
BaseAgent,
|
|
@@ -247366,7 +247377,7 @@ function openSshProxy(ssh, target) {
|
|
|
247366
247377
|
var import_ssh2, ALLOW_UNVERIFIED_SSH_HOST_KEY_ENV = "PUBLISHER_ALLOW_UNVERIFIED_SSH_HOST_KEY", TRUTHY_ENV_VALUES, SSH_CONNECT_TIMEOUT_MS = 15000, SSH_KEEPALIVE_INTERVAL_MS = 15000;
|
|
247367
247378
|
var init_proxy = __esm(() => {
|
|
247368
247379
|
init_logger();
|
|
247369
|
-
import_ssh2 = __toESM(
|
|
247380
|
+
import_ssh2 = __toESM(require_lib9(), 1);
|
|
247370
247381
|
TRUTHY_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
247371
247382
|
});
|
|
247372
247383
|
|
|
@@ -267096,6 +267107,29 @@ var require_lunr = __commonJS((exports, module) => {
|
|
|
267096
267107
|
})();
|
|
267097
267108
|
});
|
|
267098
267109
|
|
|
267110
|
+
// src/service/authorize_routes.ts
|
|
267111
|
+
function canonicalAuthorizeRoute(route) {
|
|
267112
|
+
switch (route) {
|
|
267113
|
+
case ACCESS_FILTER_ROUTE:
|
|
267114
|
+
return ACCESS_FILTER_ROUTE;
|
|
267115
|
+
case AUTHORIZE_ROUTE:
|
|
267116
|
+
return AUTHORIZE_ROUTE;
|
|
267117
|
+
default:
|
|
267118
|
+
return;
|
|
267119
|
+
}
|
|
267120
|
+
}
|
|
267121
|
+
var AUTHORIZE_ROUTE = "authorize", ACCESS_FILTER_ROUTE = "access_filter", RECOGNIZED_AUTHORIZE_SPELLINGS, CANONICAL_AUTHORIZE_ROUTES;
|
|
267122
|
+
var init_authorize_routes = __esm(() => {
|
|
267123
|
+
RECOGNIZED_AUTHORIZE_SPELLINGS = [
|
|
267124
|
+
ACCESS_FILTER_ROUTE,
|
|
267125
|
+
AUTHORIZE_ROUTE
|
|
267126
|
+
];
|
|
267127
|
+
CANONICAL_AUTHORIZE_ROUTES = [
|
|
267128
|
+
ACCESS_FILTER_ROUTE,
|
|
267129
|
+
AUTHORIZE_ROUTE
|
|
267130
|
+
];
|
|
267131
|
+
});
|
|
267132
|
+
|
|
267099
267133
|
// src/service/authorize.ts
|
|
267100
267134
|
import { payloadOf, routeOf } from "@malloydata/malloy";
|
|
267101
267135
|
function noteRoute(text) {
|
|
@@ -267104,8 +267138,61 @@ function noteRoute(text) {
|
|
|
267104
267138
|
function notePayload(text) {
|
|
267105
267139
|
return payloadOf({ value: text.trimStart() }) ?? "";
|
|
267106
267140
|
}
|
|
267141
|
+
function nearMissRouteNames(route) {
|
|
267142
|
+
switch (route) {
|
|
267143
|
+
case AUTHORIZE_ROUTE:
|
|
267144
|
+
return [
|
|
267145
|
+
"authorize",
|
|
267146
|
+
"source_authorize",
|
|
267147
|
+
"source-authorize",
|
|
267148
|
+
"sourceauthorize",
|
|
267149
|
+
"authorize_source",
|
|
267150
|
+
"authorize-source",
|
|
267151
|
+
"authorizesource"
|
|
267152
|
+
];
|
|
267153
|
+
case ACCESS_FILTER_ROUTE:
|
|
267154
|
+
return [
|
|
267155
|
+
"access_filter",
|
|
267156
|
+
"access-filter",
|
|
267157
|
+
"accessfilter",
|
|
267158
|
+
"filter_access",
|
|
267159
|
+
"filter-access",
|
|
267160
|
+
"filteraccess",
|
|
267161
|
+
"row_authorize",
|
|
267162
|
+
"row-authorize",
|
|
267163
|
+
"rowauthorize",
|
|
267164
|
+
"authorize_row",
|
|
267165
|
+
"authorize-row",
|
|
267166
|
+
"authorizerow"
|
|
267167
|
+
];
|
|
267168
|
+
default:
|
|
267169
|
+
return [route];
|
|
267170
|
+
}
|
|
267171
|
+
}
|
|
267172
|
+
function nearMissWordAlternation(route) {
|
|
267173
|
+
const names = nearMissRouteNames(route);
|
|
267174
|
+
if (names.length === 0) {
|
|
267175
|
+
throw new Error(`No near-miss spellings for authorize route \`${route}\` — an empty ` + "alternation matches every malformed-prefix note, which would " + "fail package loads over annotations nobody wrote.");
|
|
267176
|
+
}
|
|
267177
|
+
return names.map((name) => name.replace(REGEXP_SPECIAL, "\\$&")).join("|");
|
|
267178
|
+
}
|
|
267179
|
+
function motlyAuthorizePayloadPattern(route) {
|
|
267180
|
+
return new RegExp(`^[ \\t]*[([{<][ \\t]*(?:${nearMissWordAlternation(route)})[ \\t]*[)\\]}>]`, "iu");
|
|
267181
|
+
}
|
|
267182
|
+
function malformedAuthorizeAttemptPattern(route) {
|
|
267183
|
+
return new RegExp(`^##?\\|?[ \\t]*[([{<]?[ \\t]*(?:${nearMissWordAlternation(route)})`, "iu");
|
|
267184
|
+
}
|
|
267107
267185
|
function authorizeNoteContent(text) {
|
|
267108
|
-
|
|
267186
|
+
const route = canonicalAuthorizeRoute(noteRoute(text));
|
|
267187
|
+
if (route === undefined)
|
|
267188
|
+
return;
|
|
267189
|
+
return { route, content: notePayload(text) };
|
|
267190
|
+
}
|
|
267191
|
+
function authorizeAnnotationRoute(text) {
|
|
267192
|
+
return authorizeNoteContent(text)?.route;
|
|
267193
|
+
}
|
|
267194
|
+
function hasCallerAuthorizeAnnotation(callerText) {
|
|
267195
|
+
return AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText);
|
|
267109
267196
|
}
|
|
267110
267197
|
function assertNoCallerAuthorizeAnnotation(callerText) {
|
|
267111
267198
|
if (!AUTHORIZE_ANNOTATION_ANYWHERE.test(callerText))
|
|
@@ -267115,37 +267202,59 @@ function assertNoCallerAuthorizeAnnotation(callerText) {
|
|
|
267115
267202
|
function containsAuthorizeAnnotationTag(texts) {
|
|
267116
267203
|
return texts.some((text) => authorizeNoteContent(text) !== undefined);
|
|
267117
267204
|
}
|
|
267118
|
-
function collectAuthorizeNearMisses(texts) {
|
|
267205
|
+
function collectAuthorizeNearMisses(texts, route) {
|
|
267206
|
+
const motlyPattern = motlyAuthorizePayloadPattern(route);
|
|
267207
|
+
const malformedPattern = malformedAuthorizeAttemptPattern(route);
|
|
267208
|
+
const nearMissNames = new Set(nearMissRouteNames(route).map((name) => name.toLowerCase()));
|
|
267119
267209
|
const found = [];
|
|
267120
267210
|
for (const text of texts) {
|
|
267121
267211
|
const trimmed = text.trimStart();
|
|
267122
|
-
const
|
|
267123
|
-
if (
|
|
267212
|
+
const noted = noteRoute(trimmed);
|
|
267213
|
+
if (noted === route)
|
|
267124
267214
|
continue;
|
|
267125
|
-
const nearMiss =
|
|
267215
|
+
const nearMiss = noted === undefined ? malformedPattern.test(trimmed) : noted === "" ? motlyPattern.test(notePayload(trimmed)) : nearMissNames.has(noted.toLowerCase());
|
|
267126
267216
|
if (!nearMiss)
|
|
267127
267217
|
continue;
|
|
267128
267218
|
found.push(trimmed.split(/[\r\n]/, 1)[0]);
|
|
267129
267219
|
}
|
|
267130
267220
|
return found;
|
|
267131
267221
|
}
|
|
267222
|
+
function collectAuthorizeNearMissesAllRoutes(texts) {
|
|
267223
|
+
const materialized = [...texts];
|
|
267224
|
+
return RECOGNIZED_AUTHORIZE_SPELLINGS.flatMap((route) => collectAuthorizeNearMisses(materialized, route));
|
|
267225
|
+
}
|
|
267226
|
+
function guessedNearMissRoute(text) {
|
|
267227
|
+
const lower = text.toLowerCase();
|
|
267228
|
+
let best = ACCESS_FILTER_ROUTE;
|
|
267229
|
+
let bestLength = 0;
|
|
267230
|
+
for (const route of RECOGNIZED_AUTHORIZE_SPELLINGS) {
|
|
267231
|
+
for (const name of nearMissRouteNames(route)) {
|
|
267232
|
+
if (lower.includes(name) && name.length > bestLength) {
|
|
267233
|
+
best = route;
|
|
267234
|
+
bestLength = name.length;
|
|
267235
|
+
}
|
|
267236
|
+
}
|
|
267237
|
+
}
|
|
267238
|
+
return best;
|
|
267239
|
+
}
|
|
267132
267240
|
function assertNoAuthorizeNearMisses(found) {
|
|
267133
267241
|
if (found.length === 0)
|
|
267134
267242
|
return;
|
|
267135
267243
|
const unique = [...new Set(found)];
|
|
267136
267244
|
throw new ModelCompilationError({
|
|
267137
|
-
message: `These annotations are
|
|
267138
|
-
${unique.map((t) => ` - \`${t}
|
|
267245
|
+
message: `These annotations are never enforced:
|
|
267246
|
+
${unique.map((t) => ` - \`${t}\` (meant \`#(${guessedNearMissRoute(t)})\`?)`).join(`
|
|
267139
267247
|
`)}
|
|
267140
|
-
` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\`
|
|
267248
|
+
` + `Malloy routes an annotation by its prefix, and only ` + `\`#(authorize)\` and \`#(access_filter)\` reach an access-control ` + `route (in their \`##\` and ` + `block-form \`#|(...)\` spellings too). A space after the \`#\`, spaces ` + `inside the brackets, a mis-hyphenated or transposed word, or anything ` + `trailing the closing bracket makes it a plain tag Malloy hands to ` + `something else. Write the tag named above, unquoted, on its own line ` + `directly above the \`source:\` statement you mean to protect (the ` + `quoted string form is retired). This is refused rather than ` + `interpreted: guessing at the intent would let publisher start ` + `enforcing a filter on a package that has been serving every row.`
|
|
267141
267249
|
});
|
|
267142
267250
|
}
|
|
267143
267251
|
function describeMisplacedAuthorizeAnnotation(f) {
|
|
267144
|
-
if (f.kind === "query")
|
|
267145
|
-
return `on query "${f.name}"`;
|
|
267146
267252
|
if (f.kind === "file")
|
|
267147
|
-
return
|
|
267148
|
-
|
|
267253
|
+
return `at the file level (\`##(${f.route})\`)`;
|
|
267254
|
+
const tag = `\`#(${f.route})\``;
|
|
267255
|
+
if (f.kind === "query")
|
|
267256
|
+
return `on query "${f.name}" (${tag})`;
|
|
267257
|
+
return `on field "${f.fieldName}" of source "${f.name}" (${tag})`;
|
|
267149
267258
|
}
|
|
267150
267259
|
function assertNoMisplacedAuthorizeAnnotations(found) {
|
|
267151
267260
|
if (found.length === 0)
|
|
@@ -267153,9 +267262,9 @@ function assertNoMisplacedAuthorizeAnnotations(found) {
|
|
|
267153
267262
|
const positions = found.map((f) => ` - ${describeMisplacedAuthorizeAnnotation(f)}`).join(`
|
|
267154
267263
|
`);
|
|
267155
267264
|
throw new ModelCompilationError({
|
|
267156
|
-
message: `An
|
|
267265
|
+
message: `An authorize annotation is never enforced at:
|
|
267157
267266
|
${positions}
|
|
267158
|
-
` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base.
|
|
267267
|
+
` + `A gate only applies where model load looks for one — a \`source:\`'s ` + `own annotation, or one it inherits from an \`extend\`/query-source ` + `base. A file-level annotation (\`##(authorize)\` / ` + `\`##(access_filter)\`) is deprecated and no longer enforced ` + `anywhere, so it always lands here: declare the same tag on each ` + `\`source:\` it was meant to protect instead. Every other ` + `position above should move — with the same tag named beside it — ` + `to the \`source:\` statement it is meant ` + `to protect.`
|
|
267159
267268
|
});
|
|
267160
267269
|
}
|
|
267161
267270
|
function referencedGivenNames(expr) {
|
|
@@ -267192,7 +267301,7 @@ function liftProbeFilterCondition(prepared, label, filterText) {
|
|
|
267192
267301
|
return lifted;
|
|
267193
267302
|
}
|
|
267194
267303
|
function gateFilterText(exprs) {
|
|
267195
|
-
return exprs.map((e) => `(${e})`).join("
|
|
267304
|
+
return exprs.map((e) => `(${e})`).join(" and ");
|
|
267196
267305
|
}
|
|
267197
267306
|
async function liftRowLevelCondition(compiler, sourceName, exprs) {
|
|
267198
267307
|
const filterText = gateFilterText(exprs);
|
|
@@ -267202,7 +267311,7 @@ async function liftRowLevelCondition(compiler, sourceName, exprs) {
|
|
|
267202
267311
|
async function validateAuthorizeProbes(compiler, options) {
|
|
267203
267312
|
const ownNotesOf = options.authorizeOwnNotes ?? new Map;
|
|
267204
267313
|
for (const [sourceName, groups] of options.authorizeMap ?? []) {
|
|
267205
|
-
for (const exprs of groups) {
|
|
267314
|
+
for (const { route, exprs } of groups) {
|
|
267206
267315
|
if (exprs.length === 0)
|
|
267207
267316
|
continue;
|
|
267208
267317
|
let condition;
|
|
@@ -267210,17 +267319,17 @@ async function validateAuthorizeProbes(compiler, options) {
|
|
|
267210
267319
|
condition = await liftRowLevelCondition(compiler, sourceName, exprs);
|
|
267211
267320
|
} catch (err) {
|
|
267212
267321
|
const detail = err instanceof Error ? err.message : String(err);
|
|
267213
|
-
const ownNotes = ownNotesOf.get(sourceName) ?? [];
|
|
267322
|
+
const ownNotes = ownNotesOf.get(sourceName)?.get(route) ?? [];
|
|
267214
267323
|
if (ownNotes.length === 0) {
|
|
267215
267324
|
options.onRowLevelGateRejected?.("entry_point_unexpressible");
|
|
267216
267325
|
options.onRowLevelGateUnexpressible?.(sourceName, detail);
|
|
267217
267326
|
continue;
|
|
267218
267327
|
}
|
|
267219
267328
|
throw new ModelCompilationError({
|
|
267220
|
-
message: `Invalid #(
|
|
267329
|
+
message: `Invalid #(${route}) annotation on source "${sourceName}" ` + `[${exprs.join(" and ")}]: ${detail}`
|
|
267221
267330
|
});
|
|
267222
267331
|
}
|
|
267223
|
-
options.onOwnRowLevelConditionCompiled?.(sourceName, condition);
|
|
267332
|
+
options.onOwnRowLevelConditionCompiled?.(sourceName, condition, route);
|
|
267224
267333
|
}
|
|
267225
267334
|
}
|
|
267226
267335
|
}
|
|
@@ -267234,59 +267343,41 @@ function unquoteLegacyGatePayload(payload) {
|
|
|
267234
267343
|
return trimmed[0] === "'" ? inner.replace(SINGLE_QUOTE_ESCAPE, "$1") : inner.replace(DOUBLE_QUOTE_ESCAPE, "$1");
|
|
267235
267344
|
}
|
|
267236
267345
|
function parseAuthorizeAnnotation(annotation) {
|
|
267237
|
-
const
|
|
267238
|
-
if (
|
|
267346
|
+
const parsed = authorizeNoteContent(annotation);
|
|
267347
|
+
if (parsed === undefined)
|
|
267239
267348
|
return null;
|
|
267240
|
-
const trimmed = content.trim();
|
|
267349
|
+
const trimmed = parsed.content.trim();
|
|
267241
267350
|
if (trimmed.length === 0) {
|
|
267242
267351
|
throw new Error("authorize annotation has an empty expression body");
|
|
267243
267352
|
}
|
|
267244
|
-
return trimmed;
|
|
267353
|
+
return { route: parsed.route, expr: trimmed };
|
|
267245
267354
|
}
|
|
267246
267355
|
function collectAuthorizeExprs(annotations) {
|
|
267247
267356
|
const exprs = [];
|
|
267248
267357
|
for (const annotation of annotations) {
|
|
267249
|
-
const
|
|
267250
|
-
if (
|
|
267251
|
-
exprs.push(
|
|
267358
|
+
const parsed = parseAuthorizeAnnotation(annotation);
|
|
267359
|
+
if (parsed !== null) {
|
|
267360
|
+
exprs.push(parsed);
|
|
267252
267361
|
}
|
|
267253
267362
|
}
|
|
267254
267363
|
return exprs;
|
|
267255
267364
|
}
|
|
267256
|
-
function
|
|
267257
|
-
|
|
267258
|
-
for (const [sourceName, notes] of authorizeOwnNotes) {
|
|
267259
|
-
if (notes.length > 1) {
|
|
267260
|
-
found.push({ sourceName, texts: notes.map((note) => note.text) });
|
|
267261
|
-
}
|
|
267262
|
-
}
|
|
267263
|
-
return found;
|
|
267264
|
-
}
|
|
267265
|
-
function assertAtMostOneAuthorizeGate(found) {
|
|
267266
|
-
if (found.length === 0)
|
|
267267
|
-
return;
|
|
267268
|
-
const positions = found.map(({ sourceName, texts }) => ` - source "${sourceName}" declares ${texts.length}:
|
|
267269
|
-
` + texts.map((t) => ` \`${t.trim().split(/[\r\n]/, 1)[0]}\``).join(`
|
|
267270
|
-
`)).join(`
|
|
267271
|
-
`);
|
|
267272
|
-
throw new ModelCompilationError({
|
|
267273
|
-
message: `A source may declare at most one \`#(authorize)\` annotation:
|
|
267274
|
-
${positions}
|
|
267275
|
-
` + `Combine multiple conditions into one expression with \`or\` instead of ` + `repeating the annotation, e.g. ` + "`#(authorize) $ROLE = 'admin' or org_id in $GROUPS`."
|
|
267276
|
-
});
|
|
267365
|
+
function collectAuthorizeExprsForRoute(annotations, route) {
|
|
267366
|
+
return collectAuthorizeExprs(annotations).filter((e) => e.route === route).map((e) => e.expr);
|
|
267277
267367
|
}
|
|
267278
267368
|
function findLegacyStringGates(authorizeOwnNotes) {
|
|
267279
267369
|
const found = [];
|
|
267280
|
-
for (const [sourceName,
|
|
267370
|
+
for (const [sourceName, notesByRoute] of authorizeOwnNotes) {
|
|
267371
|
+
const notes = [...notesByRoute.values()].flat();
|
|
267281
267372
|
const legacyNotes = notes.filter((note) => {
|
|
267282
267373
|
const content = authorizeNoteContent(note.text);
|
|
267283
|
-
return content !== undefined && isLegacyQuotedPayload(content);
|
|
267374
|
+
return content !== undefined && isLegacyQuotedPayload(content.content);
|
|
267284
267375
|
});
|
|
267285
267376
|
if (legacyNotes.length === 0)
|
|
267286
267377
|
continue;
|
|
267287
|
-
const
|
|
267288
|
-
if (
|
|
267289
|
-
found.push({ sourceName,
|
|
267378
|
+
const gates = legacyNotes.flatMap((note) => collectAuthorizeExprs([note.text]));
|
|
267379
|
+
if (gates.length > 0) {
|
|
267380
|
+
found.push({ sourceName, gates });
|
|
267290
267381
|
}
|
|
267291
267382
|
}
|
|
267292
267383
|
return found;
|
|
@@ -267294,22 +267385,22 @@ function findLegacyStringGates(authorizeOwnNotes) {
|
|
|
267294
267385
|
function assertNoLegacyStringGate(found) {
|
|
267295
267386
|
if (found.length === 0)
|
|
267296
267387
|
return;
|
|
267297
|
-
const rewrites = found.flatMap(({ sourceName,
|
|
267298
|
-
#(
|
|
267388
|
+
const rewrites = found.flatMap(({ sourceName, gates }) => gates.map(({ route, expr }) => ` - source "${sourceName}": replace \`#(${route}) ${expr}\`` + ` with
|
|
267389
|
+
#(${route}) ${unquoteLegacyGatePayload(expr)}`)).join(`
|
|
267299
267390
|
`);
|
|
267300
267391
|
throw new ModelCompilationError({
|
|
267301
|
-
message: `The string form of
|
|
267392
|
+
message: `The string form of an authorize gate (a Malloy-quoted expression on ` + `the \`source:\` line) is no longer accepted. Replace it with the ` + `unquoted expression, carried by the same annotation on its own line ` + `directly above the \`source:\` line:
|
|
267302
267393
|
${rewrites}
|
|
267303
267394
|
` + `Test that the rewrite gates the same rows and check the row count ` + `matches what the string form served.`
|
|
267304
267395
|
});
|
|
267305
267396
|
}
|
|
267306
|
-
var
|
|
267397
|
+
var AUTHORIZE_TAG_LIKE, AUTHORIZE_ANNOTATION_ANYWHERE, REGEXP_SPECIAL, GIVEN_REF_PATTERN, SINGLE_QUOTE_ESCAPE, DOUBLE_QUOTE_ESCAPE, STRING_LITERAL_PATTERN, everyMemberOf = () => (...members) => members, ROW_LEVEL_GATE_REJECTION_CAUSES, WHOLE_BODY_QUOTED_STRING, WHOLE_BODY_SINGLE_QUOTED_STRING;
|
|
267307
267398
|
var init_authorize = __esm(() => {
|
|
267308
267399
|
init_errors();
|
|
267309
|
-
|
|
267400
|
+
init_authorize_routes();
|
|
267401
|
+
AUTHORIZE_TAG_LIKE = String.raw`##?\|?[ \t]*[([{<]?[ \t]*(?:(?:(?:row|source)[-_]?)?authorize|access[-_]?filter)(?=[)\]}>]|[ \t]|$)`;
|
|
267310
267402
|
AUTHORIZE_ANNOTATION_ANYWHERE = new RegExp(AUTHORIZE_TAG_LIKE, "iu");
|
|
267311
|
-
|
|
267312
|
-
MALFORMED_AUTHORIZE_ATTEMPT = /^##?\|?[ \t]*[([{<]?[ \t]*authorize/iu;
|
|
267403
|
+
REGEXP_SPECIAL = /[.*+?^${}()|[\]\\]/g;
|
|
267313
267404
|
GIVEN_REF_PATTERN = /\$([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
267314
267405
|
SINGLE_QUOTE_ESCAPE = /\\(['\\])/g;
|
|
267315
267406
|
DOUBLE_QUOTE_ESCAPE = /\\(["\\])/g;
|
|
@@ -267894,7 +267985,13 @@ function collectGivenRefs(value, into) {
|
|
|
267894
267985
|
for (const child of Object.values(node))
|
|
267895
267986
|
collectGivenRefs(child, into);
|
|
267896
267987
|
}
|
|
267897
|
-
function
|
|
267988
|
+
function gateGivenSource(sources, name) {
|
|
267989
|
+
const source = sources.find((candidate) => candidate.name === name);
|
|
267990
|
+
if (!source)
|
|
267991
|
+
return;
|
|
267992
|
+
return [...source.authorize ?? [], ...source.accessFilter ?? []];
|
|
267993
|
+
}
|
|
267994
|
+
function suggestGivenLookup(modelDef, gatesBySource, surfaced) {
|
|
267898
267995
|
const registry = modelDef.givens ?? {};
|
|
267899
267996
|
const bySource = new Map;
|
|
267900
267997
|
const byQuery = new Map;
|
|
@@ -267903,7 +268000,7 @@ function suggestGivenLookup(modelDef, authorizeBySource, surfaced) {
|
|
|
267903
268000
|
const name = obj.as || obj.name;
|
|
267904
268001
|
const refs = new Set;
|
|
267905
268002
|
collectGivenRefs(obj.filterList, refs);
|
|
267906
|
-
for (const expr of
|
|
268003
|
+
for (const expr of gatesBySource(name) ?? []) {
|
|
267907
268004
|
for (const given of referencedGivenNames(expr))
|
|
267908
268005
|
refs.add(given);
|
|
267909
268006
|
}
|
|
@@ -268002,7 +268099,7 @@ function positiveInteger(value) {
|
|
|
268002
268099
|
function normalizeTileExpression(tile) {
|
|
268003
268100
|
return tile.split("->").map((part) => part.trim().replace(/\s+/g, " ")).join(" -> ").replace(/\s+/g, " ");
|
|
268004
268101
|
}
|
|
268005
|
-
function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames,
|
|
268102
|
+
function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, gatesBySource) {
|
|
268006
268103
|
const registry = modelDef.givens ?? {};
|
|
268007
268104
|
const surfaced = new Set(surfacedGivenNames);
|
|
268008
268105
|
const givens = new Map;
|
|
@@ -268072,7 +268169,7 @@ function readDashboardModelFacts(modelPath, modelDef, surfacedGivenNames, author
|
|
|
268072
268169
|
viewAnnotations,
|
|
268073
268170
|
sourceFields,
|
|
268074
268171
|
drills,
|
|
268075
|
-
suggestGivens: suggestGivenLookup(modelDef, (source) =>
|
|
268172
|
+
suggestGivens: suggestGivenLookup(modelDef, (source) => gatesBySource?.get(source), surfaced)
|
|
268076
268173
|
};
|
|
268077
268174
|
}
|
|
268078
268175
|
function readTileLayout(annotations) {
|
|
@@ -270706,20 +270803,33 @@ function recordAuthorizeBypass(entryPoint) {
|
|
|
270706
270803
|
}
|
|
270707
270804
|
function recordRowLevelGateDecision(decision) {
|
|
270708
270805
|
rowLevelDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_total", {
|
|
270709
|
-
description: "How a row-level `#(
|
|
270806
|
+
description: "How a row-level `#(access_filter)` gate resolved a request. Label: decision ('denied_by_gate'|'empty_after_filter'). 'denied_by_gate' is the fail-closed refusal when the gate could not be applied; 'empty_after_filter' is a successful response with zero rows after the filter matched none, which is NOT an error."
|
|
270710
270807
|
});
|
|
270711
270808
|
rowLevelDecisionCounter.add(1, { decision });
|
|
270712
270809
|
}
|
|
270713
270810
|
function recordRowLevelGateRejected(cause) {
|
|
270714
270811
|
rowLevelRejectionCounter ??= publisherMeter().createCounter("publisher_authorize_row_level_rejected_total", {
|
|
270715
|
-
description: "Row-level `#(
|
|
270812
|
+
description: "Row-level `#(access_filter)` gates that were refused, warned about, or could not be resolved. Label: cause (" + ROW_LEVEL_GATE_REJECTION_CAUSES.map((c) => `'${c}'`).join("|") + "). Only 'legacy_string_gate' fails the whole model load. 'entry_point_unexpressible', 'source_line_gate_no_given_reference' and 'source_line_gate_negated_membership' warn at load and leave the model servable. 'unreachable_given', 'given_usage_unresolvable' and 'unclassifiable_condition' are request-time only — that entry point denies every request. Alert on any nonzero value since the last publish; the request-time causes also make this a rate signal, so see the doc above before writing an alert."
|
|
270716
270813
|
});
|
|
270717
270814
|
rowLevelRejectionCounter.add(1, { cause });
|
|
270718
270815
|
}
|
|
270719
|
-
|
|
270816
|
+
function recordAuthorizeAdmitAllGate(route) {
|
|
270817
|
+
admitAllCounter ??= publisherMeter().createCounter("publisher_authorize_admit_all_total", {
|
|
270818
|
+
description: "Sources declaring their OWN unconditional admit-all gate (`#(authorize) true`), counted once each at package load. Label: route (" + CANONICAL_AUTHORIZE_ROUTES.map((r) => `'${r}'`).join("|") + "). Not an error — `true` is the only spelling for an extension that deliberately re-opens a gated base — but it is the one declaration that turns a gate off, so a jump since the last publish is worth a look."
|
|
270819
|
+
});
|
|
270820
|
+
admitAllCounter.add(1, { route });
|
|
270821
|
+
}
|
|
270822
|
+
function recordLockDecision(decision) {
|
|
270823
|
+
lockDecisionCounter ??= publisherMeter().createCounter("publisher_authorize_lock_total", {
|
|
270824
|
+
description: "How a `#(authorize)` lock resolved a request. Label: decision ('admitted'|'denied_by_lock'|'denied_unresolvable'). Both denials are a 403; 'denied_by_lock' is the gate refusing a caller it does not admit (routine), 'denied_unresolvable' is the fail-closed refusal when the gate could not be decided at all."
|
|
270825
|
+
});
|
|
270826
|
+
lockDecisionCounter.add(1, { decision });
|
|
270827
|
+
}
|
|
270828
|
+
var guardRejectionCounter = null, bypassCounter = null, rowLevelDecisionCounter = null, rowLevelRejectionCounter = null, admitAllCounter = null, lockDecisionCounter = null;
|
|
270720
270829
|
var init_authorize_metrics = __esm(() => {
|
|
270721
270830
|
init_telemetry();
|
|
270722
270831
|
init_authorize();
|
|
270832
|
+
init_authorize_routes();
|
|
270723
270833
|
});
|
|
270724
270834
|
|
|
270725
270835
|
// src/materialization_metrics.ts
|
|
@@ -271918,6 +272028,317 @@ function filterFreshManifest(entries, now) {
|
|
|
271918
272028
|
return { manifest, nextStaleSince };
|
|
271919
272029
|
}
|
|
271920
272030
|
|
|
272031
|
+
// src/service/authorize_grammar.ts
|
|
272032
|
+
import { routeOf as routeOf2 } from "@malloydata/malloy";
|
|
272033
|
+
function noteRoute2(text) {
|
|
272034
|
+
return routeOf2({ value: text.trimStart() });
|
|
272035
|
+
}
|
|
272036
|
+
function splitFieldPathSegments(fieldPath) {
|
|
272037
|
+
const segments = [];
|
|
272038
|
+
let i = 0;
|
|
272039
|
+
while (i < fieldPath.length) {
|
|
272040
|
+
if (fieldPath[i] === "`") {
|
|
272041
|
+
const end = fieldPath.indexOf("`", i + 1);
|
|
272042
|
+
segments.push(fieldPath.slice(i + 1, end));
|
|
272043
|
+
i = end + 1;
|
|
272044
|
+
} else {
|
|
272045
|
+
const dot = fieldPath.indexOf(".", i);
|
|
272046
|
+
const end = dot === -1 ? fieldPath.length : dot;
|
|
272047
|
+
segments.push(fieldPath.slice(i, end));
|
|
272048
|
+
i = end;
|
|
272049
|
+
}
|
|
272050
|
+
if (fieldPath[i] === ".")
|
|
272051
|
+
i++;
|
|
272052
|
+
}
|
|
272053
|
+
return segments;
|
|
272054
|
+
}
|
|
272055
|
+
function rejectionMessage(sourceName, body, detail) {
|
|
272056
|
+
return `Source "${sourceName}" declares an authorize gate \`${body}\`: ${detail} ` + GRAMMAR_SUMMARY;
|
|
272057
|
+
}
|
|
272058
|
+
function reject(sourceName, body, cause, detail) {
|
|
272059
|
+
throw new AuthorizeGrammarError(cause, rejectionMessage(sourceName, body, detail));
|
|
272060
|
+
}
|
|
272061
|
+
function rejectCoherence(sourceName, cause, detail) {
|
|
272062
|
+
throw new AuthorizeGrammarError(cause, `Source "${sourceName}" declares authorize notes whose terms ` + `conflict: ${detail} ${GRAMMAR_SUMMARY}`);
|
|
272063
|
+
}
|
|
272064
|
+
function splitTerms(body) {
|
|
272065
|
+
const terms = [];
|
|
272066
|
+
const compoundTokens = [];
|
|
272067
|
+
let depth = 0;
|
|
272068
|
+
let quote;
|
|
272069
|
+
let start = 0;
|
|
272070
|
+
let i = 0;
|
|
272071
|
+
while (i < body.length) {
|
|
272072
|
+
const ch = body[i];
|
|
272073
|
+
if (quote) {
|
|
272074
|
+
if (ch === "\\" && quote !== "`") {
|
|
272075
|
+
i += 2;
|
|
272076
|
+
continue;
|
|
272077
|
+
}
|
|
272078
|
+
if (ch === quote)
|
|
272079
|
+
quote = undefined;
|
|
272080
|
+
i++;
|
|
272081
|
+
continue;
|
|
272082
|
+
}
|
|
272083
|
+
if (isQuote(ch)) {
|
|
272084
|
+
quote = ch;
|
|
272085
|
+
i++;
|
|
272086
|
+
continue;
|
|
272087
|
+
}
|
|
272088
|
+
if (ch === "(") {
|
|
272089
|
+
depth++;
|
|
272090
|
+
i++;
|
|
272091
|
+
continue;
|
|
272092
|
+
}
|
|
272093
|
+
if (ch === ")") {
|
|
272094
|
+
depth = Math.max(0, depth - 1);
|
|
272095
|
+
i++;
|
|
272096
|
+
continue;
|
|
272097
|
+
}
|
|
272098
|
+
if (depth === 0) {
|
|
272099
|
+
const wordMatch = /^(and|or|not)\b/i.exec(body.slice(i));
|
|
272100
|
+
if (wordMatch && (i === 0 || !/[A-Za-z0-9_]/.test(body[i - 1]))) {
|
|
272101
|
+
const word = wordMatch[1].toLowerCase();
|
|
272102
|
+
if (word === "and") {
|
|
272103
|
+
terms.push(body.slice(start, i));
|
|
272104
|
+
i += wordMatch[0].length;
|
|
272105
|
+
start = i;
|
|
272106
|
+
continue;
|
|
272107
|
+
}
|
|
272108
|
+
compoundTokens.push(word);
|
|
272109
|
+
}
|
|
272110
|
+
}
|
|
272111
|
+
i++;
|
|
272112
|
+
}
|
|
272113
|
+
terms.push(body.slice(start));
|
|
272114
|
+
return { terms, compoundTokens };
|
|
272115
|
+
}
|
|
272116
|
+
function isQuote(ch) {
|
|
272117
|
+
return ch === "'" || ch === '"' || ch === "`";
|
|
272118
|
+
}
|
|
272119
|
+
function findOperatorOutsideQuotes(s, re) {
|
|
272120
|
+
let quote;
|
|
272121
|
+
for (let i = 0;i < s.length; i++) {
|
|
272122
|
+
const ch = s[i];
|
|
272123
|
+
if (quote) {
|
|
272124
|
+
if (ch === "\\" && quote !== "`") {
|
|
272125
|
+
i++;
|
|
272126
|
+
continue;
|
|
272127
|
+
}
|
|
272128
|
+
if (ch === quote)
|
|
272129
|
+
quote = undefined;
|
|
272130
|
+
continue;
|
|
272131
|
+
}
|
|
272132
|
+
if (isQuote(ch)) {
|
|
272133
|
+
quote = ch;
|
|
272134
|
+
continue;
|
|
272135
|
+
}
|
|
272136
|
+
re.lastIndex = i;
|
|
272137
|
+
const m = re.exec(s);
|
|
272138
|
+
if (m && m.index === i)
|
|
272139
|
+
return i;
|
|
272140
|
+
}
|
|
272141
|
+
return -1;
|
|
272142
|
+
}
|
|
272143
|
+
function parseTerm(sourceName, body, term, givenDeclaredTypes) {
|
|
272144
|
+
const trimmed2 = term.trim();
|
|
272145
|
+
if (findOperatorOutsideQuotes(trimmed2, NEGATED_OPERATOR_RE) !== -1) {
|
|
272146
|
+
reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
|
|
272147
|
+
}
|
|
272148
|
+
if (findOperatorOutsideQuotes(trimmed2, COMPARISON_OPERATOR_RE) !== -1) {
|
|
272149
|
+
reject(sourceName, body, "comparison_operator", "only `=` and `in` are allowed, not `<`/`>`/`<=`/`>=`.");
|
|
272150
|
+
}
|
|
272151
|
+
const inIdx = findOperatorOutsideQuotes(trimmed2, IN_OPERATOR_RE);
|
|
272152
|
+
const eqIdx = findOperatorOutsideQuotes(trimmed2, EQ_OPERATOR_RE);
|
|
272153
|
+
let operator;
|
|
272154
|
+
let opIdx;
|
|
272155
|
+
let opLen;
|
|
272156
|
+
if (inIdx !== -1 && (eqIdx === -1 || inIdx < eqIdx)) {
|
|
272157
|
+
operator = "in";
|
|
272158
|
+
opIdx = inIdx;
|
|
272159
|
+
opLen = 2;
|
|
272160
|
+
} else if (eqIdx !== -1) {
|
|
272161
|
+
operator = "=";
|
|
272162
|
+
opIdx = eqIdx;
|
|
272163
|
+
opLen = 1;
|
|
272164
|
+
} else {
|
|
272165
|
+
reject(sourceName, body, "malformed_body", "no `=` or `in` operator was found.");
|
|
272166
|
+
}
|
|
272167
|
+
let left = trimmed2.slice(0, opIdx).trim();
|
|
272168
|
+
let right = trimmed2.slice(opIdx + opLen).trim();
|
|
272169
|
+
if (GIVEN_REF_RE.test(left) && STRING_LITERAL_RE.test(right)) {
|
|
272170
|
+
if (operator === "in") {
|
|
272171
|
+
reject(sourceName, body, "reversed_in_operands", `\`${left} in ${right}\` reads $GIVEN in 'literal', which Malloy ` + `rejects as array-in-string membership. Write the literal ` + `first: \`${right} in ${left}\`.`);
|
|
272172
|
+
}
|
|
272173
|
+
[left, right] = [right, left];
|
|
272174
|
+
}
|
|
272175
|
+
const leftIsFieldPath = FIELD_PATH_RE.test(left);
|
|
272176
|
+
const leftIsLiteral = STRING_LITERAL_RE.test(left);
|
|
272177
|
+
if (!leftIsFieldPath && !leftIsLiteral) {
|
|
272178
|
+
reject(sourceName, body, "left_not_field_path", `\`${left}\` is neither a field path nor a string literal — one ` + "side must be a single column, a dotted join path, or a quoted " + "literal, and the other a `$GIVEN`.");
|
|
272179
|
+
}
|
|
272180
|
+
const givenMatch = GIVEN_REF_RE.exec(right);
|
|
272181
|
+
if (!givenMatch) {
|
|
272182
|
+
reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — a term compares against ` + "`$NAME`.");
|
|
272183
|
+
}
|
|
272184
|
+
const given = givenMatch[1];
|
|
272185
|
+
const declaredType = givenDeclaredTypes.get(given);
|
|
272186
|
+
if (declaredType !== undefined) {
|
|
272187
|
+
const isListType = declaredType === "array" || declaredType.endsWith("[]");
|
|
272188
|
+
if (operator === "=" && isListType) {
|
|
272189
|
+
reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a list-typed given — use \`in\`, not \`=\`.`);
|
|
272190
|
+
}
|
|
272191
|
+
if (operator === "in" && !isListType) {
|
|
272192
|
+
reject(sourceName, body, "operator_arity_mismatch", `\`$${given}\` is a scalar given — use \`=\`, not \`in\`.`);
|
|
272193
|
+
}
|
|
272194
|
+
}
|
|
272195
|
+
if (leftIsLiteral) {
|
|
272196
|
+
return { scope: "source_level", literal: left, given };
|
|
272197
|
+
}
|
|
272198
|
+
return {
|
|
272199
|
+
scope: "row_level",
|
|
272200
|
+
fieldPath: left,
|
|
272201
|
+
fieldPathSegments: splitFieldPathSegments(left),
|
|
272202
|
+
given,
|
|
272203
|
+
operator
|
|
272204
|
+
};
|
|
272205
|
+
}
|
|
272206
|
+
function assertAuthorizeGrammarTermsCoherent(sourceName, terms) {
|
|
272207
|
+
if (terms.length > 1 && terms.some(({ term }) => term.scope === "deny_all")) {
|
|
272208
|
+
rejectCoherence(sourceName, "deny_all_with_sibling", "an unconditional `false` deny-all cannot be combined with any " + "other authorize note — a deny-all admits nothing, so a " + "sibling note can never change what is served and its " + "presence is very likely a mistake.");
|
|
272209
|
+
}
|
|
272210
|
+
const byRoute = new Map;
|
|
272211
|
+
for (const { term, route } of terms) {
|
|
272212
|
+
if (term.scope === "deny_all")
|
|
272213
|
+
continue;
|
|
272214
|
+
const list = byRoute.get(route);
|
|
272215
|
+
if (list)
|
|
272216
|
+
list.push(term);
|
|
272217
|
+
else
|
|
272218
|
+
byRoute.set(route, [term]);
|
|
272219
|
+
}
|
|
272220
|
+
for (const allRouteTerms of byRoute.values()) {
|
|
272221
|
+
if (allRouteTerms.length > 1 && allRouteTerms.some((t) => t.scope === "admit_all")) {
|
|
272222
|
+
rejectCoherence(sourceName, "admit_all_with_sibling", "an unconditional `true` admit-all cannot be combined with " + "another `#(authorize)` note — the lock's notes AND " + "together, so `true and x` reduces to `x` and the admit-all " + "is dead text. To unlock a base while its rows stay " + "filtered, pair the `true` with `#(access_filter)` instead.");
|
|
272223
|
+
}
|
|
272224
|
+
const routeTerms = allRouteTerms.filter((t) => t.scope !== "admit_all" && t.scope !== "deny_all");
|
|
272225
|
+
const seenGivens = new Set;
|
|
272226
|
+
const seenFieldPaths = new Set;
|
|
272227
|
+
for (const t of routeTerms) {
|
|
272228
|
+
if (seenGivens.has(t.given)) {
|
|
272229
|
+
rejectCoherence(sourceName, "duplicate_given", `\`$${t.given}\` is used by more than one term — a given ` + "can back at most one term per route.");
|
|
272230
|
+
}
|
|
272231
|
+
seenGivens.add(t.given);
|
|
272232
|
+
if (t.scope === "row_level") {
|
|
272233
|
+
const canonicalPath = JSON.stringify(t.fieldPathSegments);
|
|
272234
|
+
if (seenFieldPaths.has(canonicalPath)) {
|
|
272235
|
+
rejectCoherence(sourceName, "duplicate_field_path", `\`${t.fieldPath}\` is used by more than one term.`);
|
|
272236
|
+
}
|
|
272237
|
+
seenFieldPaths.add(canonicalPath);
|
|
272238
|
+
}
|
|
272239
|
+
}
|
|
272240
|
+
}
|
|
272241
|
+
}
|
|
272242
|
+
function parseAuthorizeGrammarBody(sourceName, body, givenDeclaredTypes, route) {
|
|
272243
|
+
const trimmedBody = body.trim();
|
|
272244
|
+
if (trimmedBody.length === 0) {
|
|
272245
|
+
reject(sourceName, trimmedBody, "empty_body", "the expression body is empty.");
|
|
272246
|
+
}
|
|
272247
|
+
const sentinel = trimmedBody.toLowerCase();
|
|
272248
|
+
if (sentinel === "false" || sentinel === "true") {
|
|
272249
|
+
if (route !== AUTHORIZE_ROUTE) {
|
|
272250
|
+
reject(sourceName, trimmedBody, "sentinel_in_access_filter", `a bare \`${sentinel}\` is not allowed in \`#(access_filter)\` — ` + (sentinel === "false" ? "write `#(authorize) false`, which refuses the caller " + "rather than serving them zero rows." : "it filters nothing; delete the annotation, or write " + "`#(authorize) true` to re-open a locked base."));
|
|
272251
|
+
}
|
|
272252
|
+
return [{ scope: sentinel === "false" ? "deny_all" : "admit_all" }];
|
|
272253
|
+
}
|
|
272254
|
+
const { terms: rawTerms, compoundTokens } = splitTerms(trimmedBody);
|
|
272255
|
+
if (compoundTokens.length > 0) {
|
|
272256
|
+
reject(sourceName, trimmedBody, "compound_boolean", `a compound boolean (\`${compoundTokens[0]}\`) is not allowed — ` + "join terms with `and` only.");
|
|
272257
|
+
}
|
|
272258
|
+
const parsed = rawTerms.map((term) => parseTerm(sourceName, trimmedBody, term, givenDeclaredTypes));
|
|
272259
|
+
const ADMITTED_SCOPE = {
|
|
272260
|
+
[AUTHORIZE_ROUTE]: "source_level",
|
|
272261
|
+
[ACCESS_FILTER_ROUTE]: "row_level"
|
|
272262
|
+
};
|
|
272263
|
+
const admitted = ADMITTED_SCOPE[route];
|
|
272264
|
+
const offending = parsed.find((t) => t.scope !== admitted);
|
|
272265
|
+
if (offending) {
|
|
272266
|
+
const inLock = route === AUTHORIZE_ROUTE;
|
|
272267
|
+
reject(sourceName, trimmedBody, inLock ? "row_level_term_in_authorize" : "source_level_term_in_access_filter", inLock ? "a row-level term (field on the left) is not allowed in " + "`#(authorize)`, which decides whether the caller may reach " + "the source at all — move the term to `#(access_filter)`." : "a source-level term (literal on the left) is not allowed in " + "`#(access_filter)`, which decides which rows a caller may " + "see — move the term to `#(authorize)`, which refuses a " + "caller it does not admit rather than serving them zero rows.");
|
|
272268
|
+
}
|
|
272269
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, parsed.map((term) => ({ term, route })));
|
|
272270
|
+
return parsed;
|
|
272271
|
+
}
|
|
272272
|
+
function containsRetiredRouteTag(texts) {
|
|
272273
|
+
for (const text of texts) {
|
|
272274
|
+
const route = noteRoute2(text);
|
|
272275
|
+
if (route !== undefined && RETIRED_ROUTES.includes(route)) {
|
|
272276
|
+
return route;
|
|
272277
|
+
}
|
|
272278
|
+
}
|
|
272279
|
+
return;
|
|
272280
|
+
}
|
|
272281
|
+
function reachesRetiredRouteTagBelow(node, seen = new WeakSet, depth = 0) {
|
|
272282
|
+
if (depth > MAX_RETIRED_ROUTE_WALK_DEPTH) {
|
|
272283
|
+
throw new Error("retired-route IR walk exceeded max depth");
|
|
272284
|
+
}
|
|
272285
|
+
if (node === null || typeof node !== "object")
|
|
272286
|
+
return;
|
|
272287
|
+
if (seen.has(node))
|
|
272288
|
+
return;
|
|
272289
|
+
seen.add(node);
|
|
272290
|
+
if (Array.isArray(node)) {
|
|
272291
|
+
for (const item of node) {
|
|
272292
|
+
const found = reachesRetiredRouteTagBelow(item, seen, depth + 1);
|
|
272293
|
+
if (found)
|
|
272294
|
+
return found;
|
|
272295
|
+
}
|
|
272296
|
+
return;
|
|
272297
|
+
}
|
|
272298
|
+
const record = node;
|
|
272299
|
+
if (depth > 0) {
|
|
272300
|
+
for (const key of ["blockNotes", "notes"]) {
|
|
272301
|
+
const arr = record[key];
|
|
272302
|
+
if (!Array.isArray(arr))
|
|
272303
|
+
continue;
|
|
272304
|
+
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
272305
|
+
const found = containsRetiredRouteTag(texts);
|
|
272306
|
+
if (found)
|
|
272307
|
+
return found;
|
|
272308
|
+
}
|
|
272309
|
+
}
|
|
272310
|
+
for (const [key, value] of Object.entries(record)) {
|
|
272311
|
+
if (depth === 0 && key === "annotations")
|
|
272312
|
+
continue;
|
|
272313
|
+
const found = reachesRetiredRouteTagBelow(value, seen, depth + 1);
|
|
272314
|
+
if (found)
|
|
272315
|
+
return found;
|
|
272316
|
+
}
|
|
272317
|
+
return;
|
|
272318
|
+
}
|
|
272319
|
+
var AuthorizeGrammarError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", QUOTED_IDENT = "`[^`]+`", SEGMENT, FIELD_PATH_RE, GIVEN_REF_RE, STRING_LITERAL_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE, GRAMMAR_SUMMARY, IN_OPERATOR_RE, EQ_OPERATOR_RE, RETIRED_ROUTES, MAX_RETIRED_ROUTE_WALK_DEPTH = 200;
|
|
272320
|
+
var init_authorize_grammar = __esm(() => {
|
|
272321
|
+
init_errors();
|
|
272322
|
+
init_authorize_routes();
|
|
272323
|
+
AuthorizeGrammarError = class AuthorizeGrammarError extends ModelCompilationError {
|
|
272324
|
+
rejectionCause;
|
|
272325
|
+
constructor(rejectionCause, message) {
|
|
272326
|
+
super({ message });
|
|
272327
|
+
this.rejectionCause = rejectionCause;
|
|
272328
|
+
}
|
|
272329
|
+
};
|
|
272330
|
+
SEGMENT = `(?:${IDENT}|${QUOTED_IDENT})`;
|
|
272331
|
+
FIELD_PATH_RE = new RegExp(`^${SEGMENT}(?:\\.${SEGMENT})*$`);
|
|
272332
|
+
GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
|
|
272333
|
+
STRING_LITERAL_RE = /^(?:'(?:\\.|[^'\\])*'|"(?:\\.|[^"\\])*")$/;
|
|
272334
|
+
NEGATED_OPERATOR_RE = /!=/gu;
|
|
272335
|
+
COMPARISON_OPERATOR_RE = /(?:>=|<=|>|<)/gu;
|
|
272336
|
+
GRAMMAR_SUMMARY = "An authorize gate only accepts one or more `and`-joined terms, each " + "either `<column> <op> $GIVEN` (a single field or a dotted join " + "path on the left) or `'<literal>' in $GIVEN` (a literal on the " + "left), where <op> is fixed by the given's own arity: `in` for a " + "list-typed given, `=` for a scalar one.";
|
|
272337
|
+
IN_OPERATOR_RE = /\bin\b/giu;
|
|
272338
|
+
EQ_OPERATOR_RE = /=/gu;
|
|
272339
|
+
RETIRED_ROUTES = ["partition"];
|
|
272340
|
+
});
|
|
272341
|
+
|
|
271921
272342
|
// src/service/gate_dimension.ts
|
|
271922
272343
|
import {
|
|
271923
272344
|
isJoined,
|
|
@@ -271982,11 +272403,12 @@ function containsNegatedMembership(node, depth = 0) {
|
|
|
271982
272403
|
return containsNegatedMembership(n.e, depth + 1);
|
|
271983
272404
|
return false;
|
|
271984
272405
|
}
|
|
271985
|
-
function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, conditionExpr, modelDef, onWarning) {
|
|
272406
|
+
function validateSourceLineGateGivenUsage(sourceName, route, struct, refSummary, conditionExpr, modelDef, onWarning) {
|
|
272407
|
+
const gate = `#(${route}) gate`;
|
|
271986
272408
|
const expansion = expandRefSummaryGivenIds(struct, refSummary);
|
|
271987
272409
|
if (!expansion.ok) {
|
|
271988
272410
|
throw new ModelCompilationError({
|
|
271989
|
-
message:
|
|
272411
|
+
message: `The ${gate} on source "${sourceName}" references ` + `"${expansion.unresolvedPath}", which could not be resolved to a ` + `field this model can reach. An unresolvable reference is refused, ` + `not treated as referencing no given`
|
|
271990
272412
|
});
|
|
271991
272413
|
}
|
|
271992
272414
|
const givenIds = expansion.givenIds;
|
|
@@ -271994,15 +272416,15 @@ function validateSourceLineGateGivenUsage(sourceName, struct, refSummary, condit
|
|
|
271994
272416
|
const given = modelDef.givens?.[id];
|
|
271995
272417
|
if (given && (given.default !== undefined || given.defaultText !== undefined)) {
|
|
271996
272418
|
throw new ModelCompilationError({
|
|
271997
|
-
message:
|
|
272419
|
+
message: `The ${gate} on source "${sourceName}" references ` + `\`$${given.name}\`, which is declared with a default. A ` + `caller who supplies no value for \`$${given.name}\` gets ` + `that default, which can admit what the gate was meant to ` + `exclude. Declare \`$${given.name}\` with no default`
|
|
271998
272420
|
});
|
|
271999
272421
|
}
|
|
272000
272422
|
}
|
|
272001
272423
|
if (givenIds.size === 0) {
|
|
272002
|
-
onWarning?.("source_line_gate_no_given_reference",
|
|
272424
|
+
onWarning?.("source_line_gate_no_given_reference", `The ${gate} on source "${sourceName}" references no given; it is ` + `a fixed predicate, not an access rule keyed on the caller`);
|
|
272003
272425
|
}
|
|
272004
272426
|
if (containsNegatedMembership(conditionExpr)) {
|
|
272005
|
-
onWarning?.("source_line_gate_negated_membership",
|
|
272427
|
+
onWarning?.("source_line_gate_negated_membership", `The ${gate} on source "${sourceName}" negates a membership test; ` + `an empty given then matches everything instead of nothing`);
|
|
272006
272428
|
}
|
|
272007
272429
|
}
|
|
272008
272430
|
var init_gate_dimension = __esm(() => {
|
|
@@ -272030,10 +272452,10 @@ function resolveDeclaredSource(struct, modelDef) {
|
|
|
272030
272452
|
}
|
|
272031
272453
|
return sawBrokenEntry ? { kind: "unresolvable" } : { kind: "none" };
|
|
272032
272454
|
}
|
|
272033
|
-
function ancestorGateExprs(struct, modelDef, seen = new Set) {
|
|
272455
|
+
function ancestorGateExprs(struct, modelDef, route, seen = new Set) {
|
|
272034
272456
|
let inherited = struct.annotations?.inherits;
|
|
272035
272457
|
for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
|
|
272036
|
-
const exprs2 =
|
|
272458
|
+
const exprs2 = collectAuthorizeExprsForRoute(ownLevelNoteTexts(inherited), route);
|
|
272037
272459
|
if (exprs2.length > 0)
|
|
272038
272460
|
return exprs2;
|
|
272039
272461
|
inherited = inherited.inherits;
|
|
@@ -272041,15 +272463,17 @@ function ancestorGateExprs(struct, modelDef, seen = new Set) {
|
|
|
272041
272463
|
if (inherited)
|
|
272042
272464
|
return ["false"];
|
|
272043
272465
|
seen.add(struct);
|
|
272044
|
-
if (seen.size > ANCESTOR_WALK_MAX_DEPTH)
|
|
272466
|
+
if (seen.size > ANCESTOR_WALK_MAX_DEPTH) {
|
|
272045
272467
|
return ["false"];
|
|
272468
|
+
}
|
|
272046
272469
|
const declared = resolveDeclaredSource(struct, modelDef);
|
|
272047
|
-
if (declared.kind === "unresolvable")
|
|
272470
|
+
if (declared.kind === "unresolvable") {
|
|
272048
272471
|
return ["false"];
|
|
272472
|
+
}
|
|
272049
272473
|
if (declared.kind === "none" || seen.has(declared.source))
|
|
272050
272474
|
return [];
|
|
272051
|
-
const exprs =
|
|
272052
|
-
return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, seen);
|
|
272475
|
+
const exprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(declared.source.annotations), route);
|
|
272476
|
+
return exprs.length > 0 ? exprs : ancestorGateExprs(declared.source, modelDef, route, seen);
|
|
272053
272477
|
}
|
|
272054
272478
|
function resolveQuerySourceBase(struct, modelDef) {
|
|
272055
272479
|
const duck = struct;
|
|
@@ -272063,8 +272487,8 @@ function resolveCompositeResolvedBase(struct) {
|
|
|
272063
272487
|
const duck = struct;
|
|
272064
272488
|
return duck.type === "query_source" ? duck.query?.compositeResolvedSourceDef : undefined;
|
|
272065
272489
|
}
|
|
272066
|
-
function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
|
|
272067
|
-
const direct = ancestorGateExprs(struct, modelDef, new Set(seen));
|
|
272490
|
+
function effectiveAncestorGateExprs(struct, modelDef, route, seen = new Set) {
|
|
272491
|
+
const direct = ancestorGateExprs(struct, modelDef, route, new Set(seen));
|
|
272068
272492
|
if (direct.length > 0)
|
|
272069
272493
|
return [direct];
|
|
272070
272494
|
if (seen.has(struct))
|
|
@@ -272074,18 +272498,19 @@ function effectiveAncestorGateExprs(struct, modelDef, seen = new Set) {
|
|
|
272074
272498
|
const base = resolveQuerySourceBase(struct, modelDef);
|
|
272075
272499
|
if (!base) {
|
|
272076
272500
|
const duck = struct;
|
|
272077
|
-
if (duck.type === "query_source")
|
|
272501
|
+
if (duck.type === "query_source") {
|
|
272078
272502
|
groups.push(["false"]);
|
|
272503
|
+
}
|
|
272079
272504
|
} else if (!seen.has(base)) {
|
|
272080
|
-
const ownExprs =
|
|
272081
|
-
groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, seen));
|
|
272505
|
+
const ownExprs = collectAuthorizeExprsForRoute(ownLevelNoteTexts(base.annotations), route);
|
|
272506
|
+
groups.push(...ownExprs.length > 0 ? [ownExprs] : effectiveAncestorGateExprs(base, modelDef, route, seen));
|
|
272082
272507
|
}
|
|
272083
272508
|
const composite = resolveCompositeResolvedBase(struct);
|
|
272084
272509
|
if (composite && !seen.has(composite)) {
|
|
272085
272510
|
const parentOwnNotes = base ? ownLevelNotes(base.annotations) : [];
|
|
272086
272511
|
const compositeOwnNotes = ownLevelNotes(composite.annotations).filter((note) => !parentOwnNotes.includes(note));
|
|
272087
|
-
const compositeOwn =
|
|
272088
|
-
groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, seen));
|
|
272512
|
+
const compositeOwn = collectAuthorizeExprsForRoute(compositeOwnNotes.map((note) => note.text), route);
|
|
272513
|
+
groups.push(...compositeOwn.length > 0 ? [compositeOwn] : effectiveAncestorGateExprs(composite, modelDef, route, seen));
|
|
272089
272514
|
}
|
|
272090
272515
|
return groups;
|
|
272091
272516
|
}
|
|
@@ -272114,120 +272539,9 @@ var init_gate_registry_walk = __esm(() => {
|
|
|
272114
272539
|
init_authorize();
|
|
272115
272540
|
});
|
|
272116
272541
|
|
|
272117
|
-
// src/service/partition_annotation.ts
|
|
272118
|
-
import { payloadOf as payloadOf2, routeOf as routeOf2 } from "@malloydata/malloy";
|
|
272119
|
-
function noteRoute2(text) {
|
|
272120
|
-
return routeOf2({ value: text.trimStart() });
|
|
272121
|
-
}
|
|
272122
|
-
function containsPartitionAnnotationTag(texts) {
|
|
272123
|
-
return texts.some((text) => noteRoute2(text) === PARTITION_ROUTE);
|
|
272124
|
-
}
|
|
272125
|
-
function reachesPartitionTagBelow(node, seen = new WeakSet, depth = 0) {
|
|
272126
|
-
if (depth > MAX_PARTITION_IR_WALK_DEPTH) {
|
|
272127
|
-
throw new Error("partition-marker IR walk exceeded max depth");
|
|
272128
|
-
}
|
|
272129
|
-
if (node === null || typeof node !== "object")
|
|
272130
|
-
return false;
|
|
272131
|
-
if (seen.has(node))
|
|
272132
|
-
return false;
|
|
272133
|
-
seen.add(node);
|
|
272134
|
-
if (Array.isArray(node)) {
|
|
272135
|
-
return node.some((item) => reachesPartitionTagBelow(item, seen, depth + 1));
|
|
272136
|
-
}
|
|
272137
|
-
const record = node;
|
|
272138
|
-
if (depth > 0 && record.join !== undefined)
|
|
272139
|
-
return false;
|
|
272140
|
-
if (depth > 0) {
|
|
272141
|
-
for (const key of ["blockNotes", "notes"]) {
|
|
272142
|
-
const arr = record[key];
|
|
272143
|
-
if (!Array.isArray(arr))
|
|
272144
|
-
continue;
|
|
272145
|
-
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
272146
|
-
if (containsPartitionAnnotationTag(texts))
|
|
272147
|
-
return true;
|
|
272148
|
-
}
|
|
272149
|
-
}
|
|
272150
|
-
return Object.entries(record).some(([key, value]) => depth === 0 && key === "annotations" ? false : reachesPartitionTagBelow(value, seen, depth + 1));
|
|
272151
|
-
}
|
|
272152
|
-
function notePayload2(text) {
|
|
272153
|
-
return payloadOf2({ value: text.trimStart() }) ?? "";
|
|
272154
|
-
}
|
|
272155
|
-
function rejectionMessage(sourceName, body, detail) {
|
|
272156
|
-
return `Source "${sourceName}" declares \`#(partition) ${body}\`: ${detail} ` + `#(partition) only accepts \`<column> = $GIVEN\`, where <column> is a ` + `single field or a dotted join path.`;
|
|
272157
|
-
}
|
|
272158
|
-
function reject(sourceName, body, cause, detail) {
|
|
272159
|
-
throw new PartitionAnnotationError(cause, rejectionMessage(sourceName, body, detail));
|
|
272160
|
-
}
|
|
272161
|
-
function parsePartitionAnnotation(sourceName, annotationText) {
|
|
272162
|
-
if (noteRoute2(annotationText) !== PARTITION_ROUTE)
|
|
272163
|
-
return null;
|
|
272164
|
-
const body = notePayload2(annotationText).trim();
|
|
272165
|
-
if (body.length === 0) {
|
|
272166
|
-
reject(sourceName, body, "empty_body", "the expression body is empty.");
|
|
272167
|
-
}
|
|
272168
|
-
if (COMPOUND_BOOLEAN_RE.test(body)) {
|
|
272169
|
-
reject(sourceName, body, "compound_boolean", "a compound boolean (`and`/`or`/`not`) is not allowed — declare one " + "`#(partition)` marker per column.");
|
|
272170
|
-
}
|
|
272171
|
-
if (IN_OPERATOR_RE.test(body)) {
|
|
272172
|
-
reject(sourceName, body, "in_operator", "the `in` operator is not allowed.");
|
|
272173
|
-
}
|
|
272174
|
-
if (NEGATED_OPERATOR_RE.test(body)) {
|
|
272175
|
-
reject(sourceName, body, "negated_operator", "`!=` is not allowed.");
|
|
272176
|
-
}
|
|
272177
|
-
if (COMPARISON_OPERATOR_RE.test(body)) {
|
|
272178
|
-
reject(sourceName, body, "comparison_operator", "only `=` is allowed, not `<`/`>`/`<=`/`>=`.");
|
|
272179
|
-
}
|
|
272180
|
-
const eq = body.indexOf("=");
|
|
272181
|
-
if (eq === -1) {
|
|
272182
|
-
reject(sourceName, body, "malformed_body", "no `=` was found.");
|
|
272183
|
-
}
|
|
272184
|
-
const left = body.slice(0, eq).trim();
|
|
272185
|
-
const right = body.slice(eq + 1).trim();
|
|
272186
|
-
if (!FIELD_PATH_RE.test(left)) {
|
|
272187
|
-
reject(sourceName, body, "left_not_field_path", `\`${left}\` is not a field path — the left side must be a single ` + "column or a dotted join path, not an expression.");
|
|
272188
|
-
}
|
|
272189
|
-
const givenMatch = GIVEN_REF_RE.exec(right);
|
|
272190
|
-
if (!givenMatch) {
|
|
272191
|
-
reject(sourceName, body, "missing_given_reference", `\`${right}\` is not a given reference — the right side must be ` + "`$NAME`.");
|
|
272192
|
-
}
|
|
272193
|
-
return { column: left, given: givenMatch[1] };
|
|
272194
|
-
}
|
|
272195
|
-
function collectPartitionPairs(sourceName, annotationTexts2) {
|
|
272196
|
-
const pairs = [];
|
|
272197
|
-
const seenGivens = new Map;
|
|
272198
|
-
for (const text of annotationTexts2) {
|
|
272199
|
-
const pair = parsePartitionAnnotation(sourceName, text);
|
|
272200
|
-
if (pair === null)
|
|
272201
|
-
continue;
|
|
272202
|
-
const priorColumn = seenGivens.get(pair.given);
|
|
272203
|
-
if (priorColumn !== undefined) {
|
|
272204
|
-
throw new PartitionAnnotationError("duplicate_given", `Source "${sourceName}" declares \`#(partition)\` on both ` + `\`${priorColumn}\` and \`${pair.column}\` for the same given ` + `\`$${pair.given}\` — a given can back at most one partition ` + `column per source.`);
|
|
272205
|
-
}
|
|
272206
|
-
seenGivens.set(pair.given, pair.column);
|
|
272207
|
-
pairs.push(pair);
|
|
272208
|
-
}
|
|
272209
|
-
return pairs;
|
|
272210
|
-
}
|
|
272211
|
-
var PARTITION_ROUTE = "partition", MAX_PARTITION_IR_WALK_DEPTH = 200, PartitionAnnotationError, IDENT = "[A-Za-z_][A-Za-z0-9_]*", FIELD_PATH_RE, GIVEN_REF_RE, COMPOUND_BOOLEAN_RE, IN_OPERATOR_RE, NEGATED_OPERATOR_RE, COMPARISON_OPERATOR_RE;
|
|
272212
|
-
var init_partition_annotation = __esm(() => {
|
|
272213
|
-
init_errors();
|
|
272214
|
-
PartitionAnnotationError = class PartitionAnnotationError extends ModelCompilationError {
|
|
272215
|
-
rejectionCause;
|
|
272216
|
-
constructor(rejectionCause, message) {
|
|
272217
|
-
super({ message });
|
|
272218
|
-
this.rejectionCause = rejectionCause;
|
|
272219
|
-
}
|
|
272220
|
-
};
|
|
272221
|
-
FIELD_PATH_RE = new RegExp(`^${IDENT}(?:\\.${IDENT})*$`);
|
|
272222
|
-
GIVEN_REF_RE = new RegExp(`^\\$(${IDENT})$`);
|
|
272223
|
-
COMPOUND_BOOLEAN_RE = /\b(and|or|not)\b/i;
|
|
272224
|
-
IN_OPERATOR_RE = /\bin\b/i;
|
|
272225
|
-
NEGATED_OPERATOR_RE = /!=/;
|
|
272226
|
-
COMPARISON_OPERATOR_RE = /(>=|<=|>|<)/;
|
|
272227
|
-
});
|
|
272228
|
-
|
|
272229
272542
|
// src/service/gate_classification.ts
|
|
272230
272543
|
import {
|
|
272544
|
+
isJoined as isJoined2,
|
|
272231
272545
|
isSourceDef as isSourceDef5
|
|
272232
272546
|
} from "@malloydata/malloy";
|
|
272233
272547
|
function createGateClassificationDeps(givens, modelPath) {
|
|
@@ -272237,30 +272551,41 @@ function createGateClassificationDeps(givens, modelPath) {
|
|
|
272237
272551
|
modelPath
|
|
272238
272552
|
};
|
|
272239
272553
|
}
|
|
272240
|
-
function gateExprsForOwnAnnotations(struct, modelDef, excludeNotes = []) {
|
|
272554
|
+
function gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes = []) {
|
|
272241
272555
|
const ownNotes = ownLevelNotes(struct.annotations).filter((note) => !excludeNotes.includes(note));
|
|
272242
272556
|
try {
|
|
272243
|
-
const own =
|
|
272557
|
+
const own = collectAuthorizeExprsForRoute(ownNotes.map((note) => note.text), route);
|
|
272244
272558
|
if (own.length > 0) {
|
|
272245
272559
|
return { exprs: own, fromAncestor: false };
|
|
272246
272560
|
}
|
|
272247
|
-
const ancestor = ancestorGateExprs(struct, modelDef);
|
|
272561
|
+
const ancestor = ancestorGateExprs(struct, modelDef, route);
|
|
272248
272562
|
return { exprs: ancestor, fromAncestor: ancestor.length > 0 };
|
|
272249
272563
|
} catch {
|
|
272250
|
-
return {
|
|
272564
|
+
return {
|
|
272565
|
+
exprs: ["false"],
|
|
272566
|
+
fromAncestor: false
|
|
272567
|
+
};
|
|
272251
272568
|
}
|
|
272252
272569
|
}
|
|
272253
272570
|
function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate = false, entryPointStruct = struct, excludeNotes = []) {
|
|
272571
|
+
const results = [];
|
|
272572
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
272573
|
+
results.push(...collectEntryPointGatesForRoute(struct, modelDef, route, new Set(seen), treatAsOwnGate, entryPointStruct, excludeNotes));
|
|
272574
|
+
}
|
|
272575
|
+
return results;
|
|
272576
|
+
}
|
|
272577
|
+
function collectEntryPointGatesForRoute(struct, modelDef, route, seen, treatAsOwnGate, entryPointStruct, excludeNotes) {
|
|
272254
272578
|
if (!struct || !modelDef || seen.has(struct))
|
|
272255
272579
|
return [];
|
|
272256
272580
|
seen.add(struct);
|
|
272257
272581
|
const results = [];
|
|
272258
272582
|
const label = struct.as ?? struct.name;
|
|
272259
|
-
const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, modelDef, excludeNotes);
|
|
272583
|
+
const { exprs: ownExprs, fromAncestor } = gateExprsForOwnAnnotations(struct, route, modelDef, excludeNotes);
|
|
272260
272584
|
if (ownExprs.length > 0) {
|
|
272261
272585
|
results.push({
|
|
272262
272586
|
label,
|
|
272263
272587
|
exprs: ownExprs,
|
|
272588
|
+
route,
|
|
272264
272589
|
selfContained: fromAncestor || !treatAsOwnGate,
|
|
272265
272590
|
struct: entryPointStruct
|
|
272266
272591
|
});
|
|
@@ -272269,17 +272594,18 @@ function collectEntryPointGates(struct, modelDef, seen = new Set, treatAsOwnGate
|
|
|
272269
272594
|
if (duck.type === "query_source") {
|
|
272270
272595
|
const base = resolveQuerySourceBase(struct, modelDef);
|
|
272271
272596
|
if (base) {
|
|
272272
|
-
results.push(...
|
|
272597
|
+
results.push(...collectEntryPointGatesForRoute(base, modelDef, route, seen, false, entryPointStruct, []));
|
|
272273
272598
|
} else {
|
|
272274
272599
|
results.push({
|
|
272275
272600
|
label,
|
|
272276
272601
|
exprs: ["false"],
|
|
272602
|
+
route,
|
|
272277
272603
|
selfContained: true
|
|
272278
272604
|
});
|
|
272279
272605
|
}
|
|
272280
272606
|
const resolved = duck.query?.compositeResolvedSourceDef;
|
|
272281
272607
|
if (resolved) {
|
|
272282
|
-
results.push(...
|
|
272608
|
+
results.push(...collectEntryPointGatesForRoute(resolved, modelDef, route, seen, false, entryPointStruct, base ? ownLevelNotes(base.annotations) : []));
|
|
272283
272609
|
}
|
|
272284
272610
|
}
|
|
272285
272611
|
return results;
|
|
@@ -272296,7 +272622,7 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272296
272622
|
return { shape: "rejected" };
|
|
272297
272623
|
}
|
|
272298
272624
|
const filterText = gateFilterText(entry.exprs);
|
|
272299
|
-
const cacheKey = `${graftScope.cacheScope}\x00${graftTarget}\x00${filterText}`;
|
|
272625
|
+
const cacheKey = `${graftScope.cacheScope}\x00${entry.route}\x00${graftTarget}\x00${filterText}`;
|
|
272300
272626
|
let cached2 = deps.gateShapeCache.get(cacheKey);
|
|
272301
272627
|
if (!cached2) {
|
|
272302
272628
|
let condition;
|
|
@@ -272319,7 +272645,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272319
272645
|
detail: "this entry's lifted condition carries no usable expression"
|
|
272320
272646
|
};
|
|
272321
272647
|
} else if (isBareFalseLiteral(condition.e)) {
|
|
272322
|
-
classification = {
|
|
272648
|
+
classification = {
|
|
272649
|
+
shape: "row_level",
|
|
272650
|
+
givenNames: [],
|
|
272651
|
+
givenNamesById: new Map
|
|
272652
|
+
};
|
|
272323
272653
|
} else {
|
|
272324
272654
|
const targetStruct = graftScope.modelDef.contents[graftTarget];
|
|
272325
272655
|
if (!isSourceDef5(targetStruct)) {
|
|
@@ -272337,7 +272667,13 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272337
272667
|
detail: `this gate references \`${expansion.unresolvedPath}\`, which could not be resolved on the graft target`
|
|
272338
272668
|
};
|
|
272339
272669
|
} else {
|
|
272340
|
-
const
|
|
272670
|
+
const givenNamesById = new Map;
|
|
272671
|
+
for (const id of expansion.givenIds) {
|
|
272672
|
+
const name = graftScope.modelDef.givens?.[id]?.name;
|
|
272673
|
+
if (name)
|
|
272674
|
+
givenNamesById.set(id, name);
|
|
272675
|
+
}
|
|
272676
|
+
const givenNames = Array.from(givenNamesById.values());
|
|
272341
272677
|
const literalNames = referencedGivenNames(filterText);
|
|
272342
272678
|
const accountedFor = new Set(givenNames);
|
|
272343
272679
|
const unaccounted = literalNames.filter((name) => !accountedFor.has(name));
|
|
@@ -272348,7 +272684,11 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272348
272684
|
detail: "this gate references a given id that does not resolve to a name on this model"
|
|
272349
272685
|
};
|
|
272350
272686
|
} else {
|
|
272351
|
-
classification = {
|
|
272687
|
+
classification = {
|
|
272688
|
+
shape: "row_level",
|
|
272689
|
+
givenNames,
|
|
272690
|
+
givenNamesById
|
|
272691
|
+
};
|
|
272352
272692
|
}
|
|
272353
272693
|
}
|
|
272354
272694
|
}
|
|
@@ -272374,6 +272714,14 @@ async function resolveGateShape(entry, originModelDef, graftScope, deps) {
|
|
|
272374
272714
|
if (cached2.classification.shape === "rejected") {
|
|
272375
272715
|
return { shape: "rejected", cause: cached2.classification.cause };
|
|
272376
272716
|
}
|
|
272717
|
+
if (entry.route === AUTHORIZE_ROUTE) {
|
|
272718
|
+
return {
|
|
272719
|
+
shape: "lock",
|
|
272720
|
+
condition: cached2.condition,
|
|
272721
|
+
givenNames: cached2.classification.givenNames,
|
|
272722
|
+
givenNamesById: cached2.classification.givenNamesById
|
|
272723
|
+
};
|
|
272724
|
+
}
|
|
272377
272725
|
return {
|
|
272378
272726
|
shape: "row_level",
|
|
272379
272727
|
graftTarget,
|
|
@@ -272456,142 +272804,120 @@ function isBareFalseLiteral(expr) {
|
|
|
272456
272804
|
function computeGivenDeclaredTypes(givens) {
|
|
272457
272805
|
return new Map((givens ?? []).filter((g) => g.name != null && g.type != null).map((g) => [g.name, g.type]));
|
|
272458
272806
|
}
|
|
272459
|
-
function
|
|
272460
|
-
if (!struct || !modelDef || seen.has(struct))
|
|
272461
|
-
return [];
|
|
272462
|
-
seen.add(struct);
|
|
272463
|
-
const label = struct.as ?? struct.name;
|
|
272464
|
-
const own = collectPartitionPairs(label, ownLevelNotes(struct.annotations).map((note) => note.text));
|
|
272465
|
-
if (own.length > 0)
|
|
272466
|
-
return own;
|
|
272467
|
-
let inherited = struct.annotations?.inherits;
|
|
272468
|
-
for (let depth = 0;inherited && depth < ANCESTOR_WALK_MAX_DEPTH; depth++) {
|
|
272469
|
-
const pairs = collectPartitionPairs(label, ownLevelNoteTexts(inherited));
|
|
272470
|
-
if (pairs.length > 0)
|
|
272471
|
-
return pairs;
|
|
272472
|
-
inherited = inherited.inherits;
|
|
272473
|
-
}
|
|
272474
|
-
if (inherited)
|
|
272475
|
-
throw unresolvableAncestry(label);
|
|
272476
|
-
const declared = resolveDeclaredSource(struct, modelDef);
|
|
272477
|
-
if (declared.kind === "unresolvable")
|
|
272478
|
-
throw unresolvableAncestry(label);
|
|
272479
|
-
if (declared.kind === "resolved") {
|
|
272480
|
-
const pairs = resolveEntryPointPartitions(declared.source, modelDef, seen);
|
|
272481
|
-
if (pairs.length > 0)
|
|
272482
|
-
return pairs;
|
|
272483
|
-
}
|
|
272484
|
-
const queryBase = resolveQuerySourceBase(struct, modelDef);
|
|
272485
|
-
if (queryBase)
|
|
272486
|
-
return resolveEntryPointPartitions(queryBase, modelDef, seen);
|
|
272487
|
-
return [];
|
|
272488
|
-
}
|
|
272489
|
-
function unresolvableAncestry(label) {
|
|
272490
|
-
return new PartitionAnnotationError("ancestry_unresolvable", `Could not resolve whether source "${label}" carries a ` + `\`#(partition)\` marker: the derivation chain it inherits from ` + `could not be read. Denying rather than serving unfiltered.`);
|
|
272491
|
-
}
|
|
272492
|
-
function assertPartitionAnnotationsValid(modelDef) {
|
|
272807
|
+
function assertAuthorizeGrammarValid(modelDef, authorizeMap, authorizeOwnNotes, givenDeclaredTypes, onAdmitAllGate, attributedAuthorizeOwnNotes) {
|
|
272493
272808
|
if (!modelDef)
|
|
272494
272809
|
return;
|
|
272495
|
-
for (const [
|
|
272496
|
-
|
|
272497
|
-
|
|
272498
|
-
const
|
|
272499
|
-
|
|
272500
|
-
|
|
272501
|
-
|
|
272502
|
-
|
|
272503
|
-
|
|
272504
|
-
|
|
272505
|
-
|
|
272810
|
+
for (const [sourceName, groups] of authorizeMap) {
|
|
272811
|
+
const struct = modelDef.contents[sourceName];
|
|
272812
|
+
const ownGroupTerms = [];
|
|
272813
|
+
const declaredAdmitAllRoutes = [];
|
|
272814
|
+
for (const { route, exprs } of groups) {
|
|
272815
|
+
const isOwn = (authorizeOwnNotes.get(sourceName)?.get(route)?.length ?? 0) > 0;
|
|
272816
|
+
const filteredExprs = exprs.filter((expr) => (isOwn || expr !== "false") && !isLegacyQuotedPayload(expr));
|
|
272817
|
+
const groupTerms = [];
|
|
272818
|
+
for (const expr of filteredExprs) {
|
|
272819
|
+
const terms = parseAuthorizeGrammarBody(sourceName, expr, givenDeclaredTypes, route);
|
|
272820
|
+
for (const term of terms) {
|
|
272821
|
+
groupTerms.push({ term, route });
|
|
272822
|
+
if (term.scope === "admit_all" && (attributedAuthorizeOwnNotes?.get(sourceName)?.get(route)?.length ?? 0) > 0) {
|
|
272823
|
+
declaredAdmitAllRoutes.push(route);
|
|
272824
|
+
}
|
|
272825
|
+
if (term.scope === "row_level" && isSourceDef5(struct)) {
|
|
272826
|
+
assertNoFanoutFieldPath(sourceName, struct, term.fieldPath, term.fieldPathSegments);
|
|
272827
|
+
}
|
|
272506
272828
|
}
|
|
272507
|
-
continue;
|
|
272508
272829
|
}
|
|
272509
|
-
|
|
272510
|
-
|
|
272830
|
+
if (isOwn) {
|
|
272831
|
+
ownGroupTerms.push(...groupTerms);
|
|
272832
|
+
} else {
|
|
272833
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, groupTerms);
|
|
272834
|
+
}
|
|
272511
272835
|
}
|
|
272512
|
-
if (
|
|
272513
|
-
|
|
272836
|
+
if (ownGroupTerms.length > 0) {
|
|
272837
|
+
assertAuthorizeGrammarTermsCoherent(sourceName, ownGroupTerms);
|
|
272514
272838
|
}
|
|
272515
|
-
const
|
|
272516
|
-
|
|
272517
|
-
throw new PartitionAnnotationError("partitioned_composite", `Source "${label}" is a composite source (\`compose(...)\`) with ` + `member "${marked}", which declares \`#(partition)\`. MEASURED: ` + `querying the composite reads EVERY partition of that member — ` + `the graft lands on the composite's own struct, not the ` + `resolved member branch, so no filter reaches the executed ` + `query. Remove the composite, or drop \`#(partition)\` from ` + `"${marked}" and expose it as its own source.`);
|
|
272839
|
+
for (const route of declaredAdmitAllRoutes) {
|
|
272840
|
+
onAdmitAllGate?.(sourceName, route);
|
|
272518
272841
|
}
|
|
272519
272842
|
}
|
|
272520
272843
|
}
|
|
272521
|
-
function
|
|
272522
|
-
|
|
272523
|
-
|
|
272524
|
-
|
|
272525
|
-
|
|
272526
|
-
|
|
272527
|
-
|
|
272528
|
-
|
|
272529
|
-
|
|
272530
|
-
if (!reachesBelow)
|
|
272531
|
-
return;
|
|
272532
|
-
throw new PartitionAnnotationError("marker_unreachable", `Source "${label}" carries a \`#(partition)\` marker that is not ` + `declared on the source itself — it sits on a field, a view, or an ` + `inline \`compose(...)\` inside it. Nothing would filter this ` + `source: every caller would read every partition. Move the marker ` + `onto the \`source:\` declaration.`);
|
|
272533
|
-
}
|
|
272534
|
-
function partitionedMemberLabel(composite, modelDef, seen = new Set) {
|
|
272535
|
-
if (seen.has(composite))
|
|
272536
|
-
return;
|
|
272537
|
-
seen.add(composite);
|
|
272538
|
-
for (const member of composite.sources ?? []) {
|
|
272539
|
-
const label = member.as ?? member.name;
|
|
272540
|
-
if (resolveEntryPointPartitions(member, modelDef).length > 0) {
|
|
272541
|
-
return label;
|
|
272844
|
+
function fanoutSegment(struct, path9) {
|
|
272845
|
+
let current = struct;
|
|
272846
|
+
for (let i = 0;i < path9.length - 1; i++) {
|
|
272847
|
+
const seg = path9[i];
|
|
272848
|
+
const field = (current.fields ?? []).find((f) => (f.as ?? f.name) === seg);
|
|
272849
|
+
if (!field)
|
|
272850
|
+
return { segment: seg, reason: "unresolved" };
|
|
272851
|
+
if (isJoined2(field) && field.join !== "one") {
|
|
272852
|
+
return { segment: seg, reason: "fanout" };
|
|
272542
272853
|
}
|
|
272543
|
-
if (
|
|
272544
|
-
|
|
272545
|
-
if (nested !== undefined)
|
|
272546
|
-
return nested;
|
|
272854
|
+
if (!isSourceDef5(field) && field.type !== "record" && field.type !== "array") {
|
|
272855
|
+
return { segment: seg, reason: "unresolved" };
|
|
272547
272856
|
}
|
|
272857
|
+
current = field;
|
|
272548
272858
|
}
|
|
272549
272859
|
return;
|
|
272550
272860
|
}
|
|
272551
|
-
|
|
272552
|
-
|
|
272553
|
-
|
|
272554
|
-
|
|
272555
|
-
|
|
272556
|
-
|
|
272557
|
-
|
|
272558
|
-
|
|
272559
|
-
|
|
272560
|
-
|
|
272561
|
-
|
|
272562
|
-
const
|
|
272563
|
-
if (
|
|
272564
|
-
|
|
272565
|
-
modelPath: deps.modelPath,
|
|
272566
|
-
label
|
|
272567
|
-
});
|
|
272568
|
-
throw new Error(`partition on "${label}" resolved to no graft target`);
|
|
272861
|
+
function assertNoFanoutFieldPath(sourceName, struct, fieldPath, fieldPathSegments) {
|
|
272862
|
+
const finding = fanoutSegment(struct, fieldPathSegments);
|
|
272863
|
+
if (finding === undefined)
|
|
272864
|
+
return;
|
|
272865
|
+
const detail = finding.reason === "fanout" ? `which reaches through \`${finding.segment}\`, a fan-out join ` + `(\`join_many\`/\`join_cross\`, or a repeated record) — the ` + `entry row has many values for what follows, so there is no ` + `single key to compare against a given. Gate on a field reached ` + `only through \`join_one\` joins.` : `whose segment \`${finding.segment}\` does not resolve against ` + `this source — refused rather than assumed safe.`;
|
|
272866
|
+
throw new AuthorizeGrammarError("fanout_path", `Source "${sourceName}" declares an authorize gate on \`${fieldPath}\`, ${detail}`);
|
|
272867
|
+
}
|
|
272868
|
+
function collectRetiredRouteMarkers(modelDef) {
|
|
272869
|
+
const found = [];
|
|
272870
|
+
if (!modelDef)
|
|
272871
|
+
return found;
|
|
272872
|
+
const fileMatch = containsRetiredRouteTag(ownLevelNoteTexts(modelAnnotations(modelDef)));
|
|
272873
|
+
if (fileMatch) {
|
|
272874
|
+
found.push(`the model itself declares a retired \`##(${fileMatch})\` annotation`);
|
|
272569
272875
|
}
|
|
272570
|
-
const entries
|
|
272571
|
-
|
|
272572
|
-
if (!
|
|
272573
|
-
|
|
272574
|
-
|
|
272876
|
+
for (const [key, obj] of Object.entries(modelDef.contents)) {
|
|
272877
|
+
const isQuery = obj.type === "query";
|
|
272878
|
+
if (!isSourceDef5(obj) && !isQuery)
|
|
272879
|
+
continue;
|
|
272880
|
+
const label = obj.as ?? obj.name ?? key;
|
|
272881
|
+
const noun = isQuery ? "query" : "source";
|
|
272882
|
+
const ownMatch = containsRetiredRouteTag(annotationTexts(obj.annotations) ?? []);
|
|
272883
|
+
if (ownMatch) {
|
|
272884
|
+
found.push(`${noun} "${label}" declares a retired \`#(${ownMatch})\` annotation`);
|
|
272885
|
+
continue;
|
|
272886
|
+
}
|
|
272887
|
+
if (isQuery) {
|
|
272888
|
+
continue;
|
|
272889
|
+
}
|
|
272890
|
+
let belowMatch;
|
|
272891
|
+
try {
|
|
272892
|
+
belowMatch = reachesRetiredRouteTagBelow(obj);
|
|
272893
|
+
} catch {
|
|
272894
|
+
belowMatch = RETIRED_ROUTES[0];
|
|
272895
|
+
}
|
|
272896
|
+
if (belowMatch) {
|
|
272897
|
+
found.push(`source "${label}" carries a retired \`#(${belowMatch})\` ` + `annotation somewhere in its IR (not on the source line itself)`);
|
|
272575
272898
|
}
|
|
272576
|
-
const filterText = `${pair.column} = $${pair.given}`;
|
|
272577
|
-
const condition = await liftGateCondition(graftTarget, filterText, graftScope.materializer);
|
|
272578
|
-
entries.push({
|
|
272579
|
-
label,
|
|
272580
|
-
graftTarget,
|
|
272581
|
-
filterText,
|
|
272582
|
-
condition,
|
|
272583
|
-
givenNames: [pair.given]
|
|
272584
|
-
});
|
|
272585
272899
|
}
|
|
272586
|
-
return
|
|
272900
|
+
return found;
|
|
272901
|
+
}
|
|
272902
|
+
function assertNoRetiredRouteMarkers(found) {
|
|
272903
|
+
if (found.length === 0)
|
|
272904
|
+
return;
|
|
272905
|
+
throw new ModelCompilationError({
|
|
272906
|
+
message: `These sources carry a marker from a retired annotation route:
|
|
272907
|
+
` + `${found.map((f) => ` - ${f}`).join(`
|
|
272908
|
+
`)}
|
|
272909
|
+
` + `The route is no longer enforced by anything, so Malloy would ` + `otherwise parse it, load the model clean, and serve every row as ` + `if the marker were never written. Remove the annotation.`
|
|
272910
|
+
});
|
|
272587
272911
|
}
|
|
272588
272912
|
var init_gate_classification = __esm(() => {
|
|
272913
|
+
init_errors();
|
|
272589
272914
|
init_logger();
|
|
272590
272915
|
init_annotations();
|
|
272591
272916
|
init_authorize();
|
|
272917
|
+
init_authorize_grammar();
|
|
272918
|
+
init_authorize_routes();
|
|
272592
272919
|
init_gate_dimension();
|
|
272593
272920
|
init_gate_registry_walk();
|
|
272594
|
-
init_partition_annotation();
|
|
272595
272921
|
});
|
|
272596
272922
|
|
|
272597
272923
|
// src/service/persist_dynamic_terms.ts
|
|
@@ -273061,14 +273387,7 @@ function assertMaterializationEligible(persistSource, annotationFields) {
|
|
|
273061
273387
|
recordEligibilityRefused("authorize");
|
|
273062
273388
|
throw new MaterializationEligibilityError({
|
|
273063
273389
|
reason: "authorize",
|
|
273064
|
-
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by
|
|
273065
|
-
});
|
|
273066
|
-
}
|
|
273067
|
-
if (referencesPartition(persistSource)) {
|
|
273068
|
-
recordEligibilityRefused("partition");
|
|
273069
|
-
throw new MaterializationEligibilityError({
|
|
273070
|
-
reason: "partition",
|
|
273071
|
-
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it declares a #(partition) marker. A partition ` + `filter binds a given at query time (the same mechanism as ` + `row-level access control), so a materialized-once table served ` + `frozen would leak every partition's rows to every caller. This ` + `is refused for safety. Serve this source live (drop 'storage=').`
|
|
273390
|
+
message: `Source '${sourceName}' cannot be materialized into a storage ` + `destination: it is protected by a gate — #(authorize) or ` + `#(access_filter), its own or a joined source's. Both are evaluated ` + `per request; a materialized-once table served frozen carries no ` + `gate, so it would be served to everyone, bypassing authorization. ` + `This is refused for safety. Serve this source live (drop ` + `'storage=').`
|
|
273072
273391
|
});
|
|
273073
273392
|
}
|
|
273074
273393
|
const partition = resolvePartitionColumns(persistSource, annotationFields);
|
|
@@ -273104,15 +273423,6 @@ function assertColocatedPersistNotAuthorizeGated(persistSource, sourceName = per
|
|
|
273104
273423
|
message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${partition.detail}.`
|
|
273105
273424
|
});
|
|
273106
273425
|
}
|
|
273107
|
-
if (referencesPartition(persistSource)) {
|
|
273108
|
-
recordEligibilityRefused("partition");
|
|
273109
|
-
const what2 = origin2 === "preaggregate" ? `Pre-aggregation rollup '${sourceName}'` : `Source '${sourceName}'`;
|
|
273110
|
-
const gated2 = origin2 === "preaggregate" ? `the source '${sourceName}' rolls up declares` : `it declares`;
|
|
273111
|
-
throw new MaterializationEligibilityError({
|
|
273112
|
-
reason: "partition",
|
|
273113
|
-
message: `${what2} cannot be materialized (colocated '#@ persist'): ` + `${gated2} a #(partition) marker. A partition filter binds a ` + `given at query time; this pass cannot prove the persisted ` + `artifact's read path still applies it, so this is refused for ` + `safety. Move the marker to a source that is not materialized, ` + `or stop persisting this one.`
|
|
273114
|
-
});
|
|
273115
|
-
}
|
|
273116
273426
|
if (buildSubstitutesAGiven(persistSource)) {
|
|
273117
273427
|
if (origin2 === "preaggregate") {
|
|
273118
273428
|
recordEligibilityRefused("preaggregate_over_dynamic_source");
|
|
@@ -273163,44 +273473,6 @@ function unboundParameterNames(persistSource) {
|
|
|
273163
273473
|
}
|
|
273164
273474
|
return unbound;
|
|
273165
273475
|
}
|
|
273166
|
-
function referencesPartition(persistSource) {
|
|
273167
|
-
try {
|
|
273168
|
-
return walkForPartition(persistSource._sourceDef, new WeakSet, 0);
|
|
273169
|
-
} catch {
|
|
273170
|
-
return true;
|
|
273171
|
-
}
|
|
273172
|
-
}
|
|
273173
|
-
function walkForPartition(node, seen, depth) {
|
|
273174
|
-
if (depth > MAX_GIVEN_WALK_DEPTH) {
|
|
273175
|
-
throw new Error("partition-usage walk exceeded max depth");
|
|
273176
|
-
}
|
|
273177
|
-
if (node === null || typeof node !== "object")
|
|
273178
|
-
return false;
|
|
273179
|
-
if (seen.has(node))
|
|
273180
|
-
return false;
|
|
273181
|
-
seen.add(node);
|
|
273182
|
-
if (Array.isArray(node)) {
|
|
273183
|
-
for (const item of node) {
|
|
273184
|
-
if (walkForPartition(item, seen, depth + 1))
|
|
273185
|
-
return true;
|
|
273186
|
-
}
|
|
273187
|
-
return false;
|
|
273188
|
-
}
|
|
273189
|
-
const record = node;
|
|
273190
|
-
for (const key of ["blockNotes", "notes"]) {
|
|
273191
|
-
const arr = record[key];
|
|
273192
|
-
if (!Array.isArray(arr))
|
|
273193
|
-
continue;
|
|
273194
|
-
const texts = arr.map((n) => typeof n === "string" ? n : n && typeof n === "object" && typeof n.text === "string" ? n.text : undefined).filter((text) => text !== undefined);
|
|
273195
|
-
if (containsPartitionAnnotationTag(texts))
|
|
273196
|
-
return true;
|
|
273197
|
-
}
|
|
273198
|
-
for (const value of Object.values(record)) {
|
|
273199
|
-
if (walkForPartition(value, seen, depth + 1))
|
|
273200
|
-
return true;
|
|
273201
|
-
}
|
|
273202
|
-
return false;
|
|
273203
|
-
}
|
|
273204
273476
|
function referencesAuthorize(persistSource) {
|
|
273205
273477
|
try {
|
|
273206
273478
|
return walkForAuthorize(persistSource._sourceDef, new WeakSet, 0);
|
|
@@ -273302,7 +273574,6 @@ var init_materialization_eligibility = __esm(() => {
|
|
|
273302
273574
|
init_materialization_metrics();
|
|
273303
273575
|
init_authorize();
|
|
273304
273576
|
init_build_plan();
|
|
273305
|
-
init_partition_annotation();
|
|
273306
273577
|
init_persist_dynamic_terms();
|
|
273307
273578
|
});
|
|
273308
273579
|
|
|
@@ -274237,7 +274508,7 @@ function buildDerivationBaseMap(text) {
|
|
|
274237
274508
|
import {
|
|
274238
274509
|
expressionIsAggregate,
|
|
274239
274510
|
expressionIsAnalytic,
|
|
274240
|
-
isJoined as
|
|
274511
|
+
isJoined as isJoined3
|
|
274241
274512
|
} from "@malloydata/malloy";
|
|
274242
274513
|
function isAccessRestricted2(f) {
|
|
274243
274514
|
return f.accessModifier != null && f.accessModifier !== "public";
|
|
@@ -274249,13 +274520,13 @@ function isView(f) {
|
|
|
274249
274520
|
return f.type === "turtle";
|
|
274250
274521
|
}
|
|
274251
274522
|
function isFanoutJoin(f) {
|
|
274252
|
-
return
|
|
274523
|
+
return isJoined3(f) && f.join !== "one";
|
|
274253
274524
|
}
|
|
274254
274525
|
function isMeasure(f) {
|
|
274255
274526
|
return expressionIsAggregate(f.expressionType) && !expressionIsAnalytic(f.expressionType);
|
|
274256
274527
|
}
|
|
274257
274528
|
function isGroupableDimension(f) {
|
|
274258
|
-
if (
|
|
274529
|
+
if (isJoined3(f) || isView(f))
|
|
274259
274530
|
return false;
|
|
274260
274531
|
if (f.expressionType === undefined)
|
|
274261
274532
|
return true;
|
|
@@ -274275,7 +274546,7 @@ function validateGrainDimension(sourceName, measure, source, grainDimension) {
|
|
|
274275
274546
|
if (!field) {
|
|
274276
274547
|
return violation("unknown_grain_dimension", `Measure \`${measure}\` declares \`#@ preaggregate\` at a grain naming \`${grainDimension}\`, but \`${head}\` is not a field of \`${sourceName}\`. Use a dimension that exists on the source; a rollup can only group by the source's own dimensions.`);
|
|
274277
274548
|
}
|
|
274278
|
-
if (
|
|
274549
|
+
if (isJoined3(field)) {
|
|
274279
274550
|
return violation("grain_path_through_join", `Measure \`${measure}\` declares a grain of \`${grainDimension}\`, which reaches through the join \`${head}\`. Pre-aggregating across a join is not supported in this version; copy the value into a dimension on \`${sourceName}\` and group by that instead.`);
|
|
274280
274551
|
}
|
|
274281
274552
|
if (parts.length > 1) {
|
|
@@ -274324,7 +274595,7 @@ function validateSourcePreaggregation(sourceName, source) {
|
|
|
274324
274595
|
if (!declaration.declared)
|
|
274325
274596
|
continue;
|
|
274326
274597
|
declaredFields.push(field);
|
|
274327
|
-
if (
|
|
274598
|
+
if (isJoined3(field)) {
|
|
274328
274599
|
violations.push({
|
|
274329
274600
|
code: "misplaced_on_join",
|
|
274330
274601
|
sourceName,
|
|
@@ -274467,7 +274738,7 @@ var init_preaggregation_validation = __esm(() => {
|
|
|
274467
274738
|
|
|
274468
274739
|
// src/service/source_extraction.ts
|
|
274469
274740
|
import {
|
|
274470
|
-
isJoined as
|
|
274741
|
+
isJoined as isJoined4,
|
|
274471
274742
|
isSourceDef as isSourceDef6,
|
|
274472
274743
|
modelDefToModelInfo
|
|
274473
274744
|
} from "@malloydata/malloy";
|
|
@@ -274521,7 +274792,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274521
274792
|
considerNoteOwner(authorizeNoteDeclaredBy, note, struct);
|
|
274522
274793
|
}
|
|
274523
274794
|
}
|
|
274524
|
-
nearMissAuthorize.push(...
|
|
274795
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
|
|
274525
274796
|
...ownLevelNotes(struct.annotations),
|
|
274526
274797
|
...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
|
|
274527
274798
|
].map((note) => note.text)));
|
|
@@ -274539,23 +274810,29 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274539
274810
|
considerNoteOwner(authorizeNoteDeclaredBy, note, entry);
|
|
274540
274811
|
}
|
|
274541
274812
|
}
|
|
274542
|
-
nearMissAuthorize.push(...
|
|
274813
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes(ownLevelNotes(entry.annotations).map((note) => note.text)));
|
|
274543
274814
|
}
|
|
274544
274815
|
for (const struct of derivedStructsReachable(sweptStructs, modelDef)) {
|
|
274545
|
-
nearMissAuthorize.push(...
|
|
274816
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([
|
|
274546
274817
|
...ownLevelNotes(struct.annotations),
|
|
274547
274818
|
...struct.fields.flatMap((field) => ownLevelNotes(field.annotations))
|
|
274548
274819
|
].map((note) => note.text)));
|
|
274549
274820
|
}
|
|
274550
274821
|
{
|
|
274551
274822
|
const folded = modelAnnotations(modelDef);
|
|
274552
|
-
nearMissAuthorize.push(...
|
|
274823
|
+
nearMissAuthorize.push(...collectAuthorizeNearMissesAllRoutes([...folded.notes ?? [], ...folded.blockNotes ?? []].map((note) => note.text)));
|
|
274553
274824
|
}
|
|
274554
274825
|
assertNoAuthorizeNearMisses(nearMissAuthorize);
|
|
274555
274826
|
const authorizeOwnNotes = new Map;
|
|
274556
274827
|
const attributedAuthorizeOwnNotes = new Map;
|
|
274557
|
-
|
|
274558
|
-
|
|
274828
|
+
{
|
|
274829
|
+
const fileNoteTexts = ownLevelNoteTexts(modelAnnotations(modelDef));
|
|
274830
|
+
if (containsAuthorizeAnnotationTag(fileNoteTexts)) {
|
|
274831
|
+
misplacedAuthorize.push({
|
|
274832
|
+
kind: "file",
|
|
274833
|
+
route: fileNoteTexts.map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274834
|
+
});
|
|
274835
|
+
}
|
|
274559
274836
|
}
|
|
274560
274837
|
const sources = Object.values(modelDef.contents).filter((obj) => isSourceDef6(obj)).map((sourceObj) => {
|
|
274561
274838
|
const struct = sourceObj;
|
|
@@ -274594,17 +274871,32 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274594
274871
|
}
|
|
274595
274872
|
}
|
|
274596
274873
|
const ownNotes = ownLevelNoteTexts(struct.annotations);
|
|
274597
|
-
const ownGates = collectAuthorizeExprs(ownNotes);
|
|
274598
274874
|
const ownAuthorizeNotes = ownLevelNotes(struct.annotations).filter((note) => containsAuthorizeAnnotationTag([note.text]));
|
|
274599
|
-
|
|
274600
|
-
|
|
274601
|
-
const
|
|
274602
|
-
|
|
274603
|
-
|
|
274604
|
-
|
|
274605
|
-
|
|
274606
|
-
|
|
274607
|
-
|
|
274875
|
+
const ownNotesByRoute = new Map;
|
|
274876
|
+
const attributedOwnNotesByRoute = new Map;
|
|
274877
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
274878
|
+
const notesForRoute = ownAuthorizeNotes.filter((note) => authorizeAnnotationRoute(note.text) === route);
|
|
274879
|
+
ownNotesByRoute.set(route, notesForRoute);
|
|
274880
|
+
attributedOwnNotesByRoute.set(route, notesForRoute.filter((note) => authorizeNoteDeclaredBy.get(note) === struct));
|
|
274881
|
+
}
|
|
274882
|
+
authorizeOwnNotes.set(sourceName, ownNotesByRoute);
|
|
274883
|
+
attributedAuthorizeOwnNotes.set(sourceName, attributedOwnNotesByRoute);
|
|
274884
|
+
const groupsForSource = [];
|
|
274885
|
+
const routeGroupsByRoute = new Map;
|
|
274886
|
+
for (const route of CANONICAL_AUTHORIZE_ROUTES) {
|
|
274887
|
+
const ownExprsForRoute = collectAuthorizeExprsForRoute(ownNotes, route);
|
|
274888
|
+
const groupsForRoute = ownExprsForRoute.length > 0 ? [ownExprsForRoute] : effectiveAncestorGateExprs(struct, modelDef, route);
|
|
274889
|
+
for (const exprs of groupsForRoute) {
|
|
274890
|
+
groupsForSource.push({ route, exprs });
|
|
274891
|
+
}
|
|
274892
|
+
routeGroupsByRoute.set(route, groupsForRoute);
|
|
274893
|
+
}
|
|
274894
|
+
if (groupsForSource.length > 0) {
|
|
274895
|
+
authorizeMap.set(sourceName, groupsForSource);
|
|
274896
|
+
}
|
|
274897
|
+
const routeGroupsToFlatWire = (groups) => groups && groups.length > 0 ? groups.flat() : undefined;
|
|
274898
|
+
const authorize = routeGroupsToFlatWire(routeGroupsByRoute.get(AUTHORIZE_ROUTE));
|
|
274899
|
+
const accessFilter = routeGroupsToFlatWire(routeGroupsByRoute.get(ACCESS_FILTER_ROUTE));
|
|
274608
274900
|
const views = struct.fields.filter((field) => field.type === "turtle").filter((turtle) => turtle.pipeline.map((stage) => stage.type).every((type) => type === "reduce")).map((turtle) => ({
|
|
274609
274901
|
name: turtle.as || turtle.name,
|
|
274610
274902
|
annotations: annotationTexts(turtle.annotations)
|
|
@@ -274617,7 +274909,7 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274617
274909
|
continue;
|
|
274618
274910
|
}
|
|
274619
274911
|
const fieldName2 = field.as || field.name;
|
|
274620
|
-
if (
|
|
274912
|
+
if (isJoined4(field) && isSourceDef6(field)) {
|
|
274621
274913
|
const joinedStruct = field;
|
|
274622
274914
|
if (joinFieldNamesUnresolvableDeclaration(joinedStruct, modelDef)) {
|
|
274623
274915
|
continue;
|
|
@@ -274625,14 +274917,16 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274625
274917
|
misplacedAuthorize.push({
|
|
274626
274918
|
kind: "field",
|
|
274627
274919
|
name: sourceName,
|
|
274628
|
-
fieldName: fieldName2
|
|
274920
|
+
fieldName: fieldName2,
|
|
274921
|
+
route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274629
274922
|
});
|
|
274630
274923
|
continue;
|
|
274631
274924
|
}
|
|
274632
274925
|
misplacedAuthorize.push({
|
|
274633
274926
|
kind: "field",
|
|
274634
274927
|
name: sourceName,
|
|
274635
|
-
fieldName: fieldName2
|
|
274928
|
+
fieldName: fieldName2,
|
|
274929
|
+
route: fieldAuthorizeNotes.map((note) => authorizeAnnotationRoute(note.text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274636
274930
|
});
|
|
274637
274931
|
}
|
|
274638
274932
|
return {
|
|
@@ -274641,7 +274935,8 @@ function extractSourcesFromModelDef(modelDef, givens, onParseError) {
|
|
|
274641
274935
|
views,
|
|
274642
274936
|
filters,
|
|
274643
274937
|
givens,
|
|
274644
|
-
authorize
|
|
274938
|
+
authorize,
|
|
274939
|
+
accessFilter
|
|
274645
274940
|
};
|
|
274646
274941
|
});
|
|
274647
274942
|
return {
|
|
@@ -274658,7 +274953,8 @@ function extractQueriesFromModelDef(modelDef) {
|
|
|
274658
274953
|
const namedQueries = Object.values(modelDef.contents).filter(isNamedQuery);
|
|
274659
274954
|
const misplacedAuthorize = namedQueries.filter((queryObj) => containsAuthorizeAnnotationTag(ownLevelNoteTexts(queryObj.annotations))).map((queryObj) => ({
|
|
274660
274955
|
kind: "query",
|
|
274661
|
-
name: queryObj.as || queryObj.name
|
|
274956
|
+
name: queryObj.as || queryObj.name,
|
|
274957
|
+
route: ownLevelNoteTexts(queryObj.annotations).map((text) => authorizeAnnotationRoute(text)).find((r) => r !== undefined) ?? ACCESS_FILTER_ROUTE
|
|
274662
274958
|
}));
|
|
274663
274959
|
const queries = namedQueries.map((queryObj) => ({
|
|
274664
274960
|
name: queryObj.as || queryObj.name,
|
|
@@ -274670,10 +274966,113 @@ function extractQueriesFromModelDef(modelDef) {
|
|
|
274670
274966
|
var init_source_extraction = __esm(() => {
|
|
274671
274967
|
init_annotations();
|
|
274672
274968
|
init_authorize();
|
|
274969
|
+
init_authorize_routes();
|
|
274673
274970
|
init_filter();
|
|
274674
274971
|
init_gate_registry_walk();
|
|
274675
274972
|
});
|
|
274676
274973
|
|
|
274974
|
+
// src/service/authorize_lock.ts
|
|
274975
|
+
function decideLock(conditionExpr, givenNameOf, givens) {
|
|
274976
|
+
const ctx = { givenNameOf, givens, unresolvable: false };
|
|
274977
|
+
try {
|
|
274978
|
+
if (admits(conditionExpr, ctx, 0))
|
|
274979
|
+
return "admit";
|
|
274980
|
+
} catch {
|
|
274981
|
+
ctx.unresolvable = true;
|
|
274982
|
+
}
|
|
274983
|
+
return ctx.unresolvable ? "unresolvable" : "deny";
|
|
274984
|
+
}
|
|
274985
|
+
function admits(node, ctx, depth) {
|
|
274986
|
+
if (depth > 64 || node === null || typeof node !== "object") {
|
|
274987
|
+
ctx.unresolvable = true;
|
|
274988
|
+
return false;
|
|
274989
|
+
}
|
|
274990
|
+
const n = node;
|
|
274991
|
+
switch (n.node) {
|
|
274992
|
+
case "()":
|
|
274993
|
+
return admits(n.e, ctx, depth + 1);
|
|
274994
|
+
case "true":
|
|
274995
|
+
return true;
|
|
274996
|
+
case "false":
|
|
274997
|
+
return false;
|
|
274998
|
+
case "and": {
|
|
274999
|
+
const left = admits(n.kids?.left, ctx, depth + 1);
|
|
275000
|
+
const right = admits(n.kids?.right, ctx, depth + 1);
|
|
275001
|
+
return left && right;
|
|
275002
|
+
}
|
|
275003
|
+
case "=": {
|
|
275004
|
+
const { left, right } = n.kids ?? {};
|
|
275005
|
+
const pair = givenAndLiteral(left, right) ?? givenAndLiteral(right, left);
|
|
275006
|
+
if (!pair) {
|
|
275007
|
+
ctx.unresolvable = true;
|
|
275008
|
+
return false;
|
|
275009
|
+
}
|
|
275010
|
+
const value = resolve4(pair.given, ctx);
|
|
275011
|
+
if (typeof value !== "string") {
|
|
275012
|
+
ctx.unresolvable = true;
|
|
275013
|
+
return false;
|
|
275014
|
+
}
|
|
275015
|
+
return value === pair.literal;
|
|
275016
|
+
}
|
|
275017
|
+
case "inGiven": {
|
|
275018
|
+
if (n.not === true)
|
|
275019
|
+
return false;
|
|
275020
|
+
const given = asGiven(n.givenRef);
|
|
275021
|
+
const literal = asStringLiteral(n.e);
|
|
275022
|
+
if (!given || literal === undefined) {
|
|
275023
|
+
ctx.unresolvable = true;
|
|
275024
|
+
return false;
|
|
275025
|
+
}
|
|
275026
|
+
const value = resolve4(given, ctx);
|
|
275027
|
+
if (!Array.isArray(value)) {
|
|
275028
|
+
ctx.unresolvable = true;
|
|
275029
|
+
return false;
|
|
275030
|
+
}
|
|
275031
|
+
return value.some((item) => item === literal);
|
|
275032
|
+
}
|
|
275033
|
+
default:
|
|
275034
|
+
ctx.unresolvable = true;
|
|
275035
|
+
return false;
|
|
275036
|
+
}
|
|
275037
|
+
}
|
|
275038
|
+
function givenAndLiteral(a, b) {
|
|
275039
|
+
const given = asGiven(a);
|
|
275040
|
+
const literal = asStringLiteral(b);
|
|
275041
|
+
if (!given || literal === undefined)
|
|
275042
|
+
return;
|
|
275043
|
+
return { given, literal };
|
|
275044
|
+
}
|
|
275045
|
+
function asGiven(node) {
|
|
275046
|
+
if (node === null || typeof node !== "object")
|
|
275047
|
+
return;
|
|
275048
|
+
const n = node;
|
|
275049
|
+
return n.node === "given" ? n : undefined;
|
|
275050
|
+
}
|
|
275051
|
+
function asStringLiteral(node) {
|
|
275052
|
+
if (node === null || typeof node !== "object")
|
|
275053
|
+
return;
|
|
275054
|
+
const n = node;
|
|
275055
|
+
if (n.node !== "stringLiteral")
|
|
275056
|
+
return;
|
|
275057
|
+
return typeof n.literal === "string" ? n.literal : undefined;
|
|
275058
|
+
}
|
|
275059
|
+
function resolve4(given, ctx) {
|
|
275060
|
+
if (typeof given.id !== "string") {
|
|
275061
|
+
ctx.unresolvable = true;
|
|
275062
|
+
return;
|
|
275063
|
+
}
|
|
275064
|
+
const name = ctx.givenNameOf(given.id);
|
|
275065
|
+
if (name === undefined) {
|
|
275066
|
+
ctx.unresolvable = true;
|
|
275067
|
+
return;
|
|
275068
|
+
}
|
|
275069
|
+
if (!Object.prototype.hasOwnProperty.call(ctx.givens, name)) {
|
|
275070
|
+
ctx.unresolvable = true;
|
|
275071
|
+
return;
|
|
275072
|
+
}
|
|
275073
|
+
return ctx.givens[name];
|
|
275074
|
+
}
|
|
275075
|
+
|
|
274677
275076
|
// src/service/model.ts
|
|
274678
275077
|
import {
|
|
274679
275078
|
API,
|
|
@@ -274695,6 +275094,22 @@ import { readFileSync as readFileSync2 } from "fs";
|
|
|
274695
275094
|
import { createRequire as createRequire2 } from "module";
|
|
274696
275095
|
import * as path10 from "path";
|
|
274697
275096
|
import { fileURLToPath as fileURLToPath6 } from "url";
|
|
275097
|
+
function denyUnlessAdmitted(resolution, givens, label) {
|
|
275098
|
+
if (resolution.shape === "lock") {
|
|
275099
|
+
const outcome = decideLock(resolution.condition.e, (id) => resolution.givenNamesById.get(id), givens);
|
|
275100
|
+
if (outcome === "admit") {
|
|
275101
|
+
recordLockDecision("admitted");
|
|
275102
|
+
return;
|
|
275103
|
+
}
|
|
275104
|
+
recordLockDecision(outcome === "unresolvable" ? "denied_unresolvable" : "denied_by_lock");
|
|
275105
|
+
throw new AccessDeniedError(`Access denied for source "${label}".`);
|
|
275106
|
+
}
|
|
275107
|
+
recordLockDecision("denied_unresolvable");
|
|
275108
|
+
recordRowLevelGateDecision("denied_by_gate");
|
|
275109
|
+
if (resolution.cause)
|
|
275110
|
+
recordRowLevelGateRejected(resolution.cause);
|
|
275111
|
+
throw new AccessDeniedError(`Access denied for source "${label}".`);
|
|
275112
|
+
}
|
|
274698
275113
|
function quoteMalloyIdentifier2(name) {
|
|
274699
275114
|
return "`" + (name ?? "").replace(/\\/g, "\\\\").replace(/`/g, "\\`") + "`";
|
|
274700
275115
|
}
|
|
@@ -274782,6 +275197,7 @@ var init_model = __esm(() => {
|
|
|
274782
275197
|
init_annotations();
|
|
274783
275198
|
init_build_plan();
|
|
274784
275199
|
init_authorize();
|
|
275200
|
+
init_authorize_routes();
|
|
274785
275201
|
init_dashboard();
|
|
274786
275202
|
init_gate_dimension();
|
|
274787
275203
|
init_filter();
|
|
@@ -274793,7 +275209,6 @@ var init_model = __esm(() => {
|
|
|
274793
275209
|
init_query_metadata();
|
|
274794
275210
|
init_preaggregation_validation();
|
|
274795
275211
|
init_gate_registry_walk();
|
|
274796
|
-
init_partition_annotation();
|
|
274797
275212
|
init_gate_classification();
|
|
274798
275213
|
init_source_extraction();
|
|
274799
275214
|
init_authorize_metrics();
|
|
@@ -274867,9 +275282,17 @@ var init_model = __esm(() => {
|
|
|
274867
275282
|
for (const source of this.sources ?? []) {
|
|
274868
275283
|
if (!source.name)
|
|
274869
275284
|
continue;
|
|
274870
|
-
const
|
|
274871
|
-
|
|
274872
|
-
|
|
275285
|
+
const gates = this.entryPointGatesBySource.get(source.name);
|
|
275286
|
+
const exprsForRoute = (route) => {
|
|
275287
|
+
const exprs = gates?.filter((g) => g.route === route).flatMap((g) => g.exprs);
|
|
275288
|
+
return exprs && exprs.length > 0 ? exprs : undefined;
|
|
275289
|
+
};
|
|
275290
|
+
const lockExprs = exprsForRoute(AUTHORIZE_ROUTE);
|
|
275291
|
+
if (lockExprs)
|
|
275292
|
+
source.authorize = lockExprs;
|
|
275293
|
+
const filterExprs = exprsForRoute(ACCESS_FILTER_ROUTE);
|
|
275294
|
+
if (filterExprs)
|
|
275295
|
+
source.accessFilter = filterExprs;
|
|
274873
275296
|
}
|
|
274874
275297
|
try {
|
|
274875
275298
|
this.authorizeReferencedGivenNames = this.computeAuthorizeReferencedGivenNames();
|
|
@@ -274940,6 +275363,9 @@ var init_model = __esm(() => {
|
|
|
274940
275363
|
getAuthorize(sourceName) {
|
|
274941
275364
|
return this.sources?.find((source) => source.name === sourceName)?.authorize ?? [];
|
|
274942
275365
|
}
|
|
275366
|
+
getAccessFilter(sourceName) {
|
|
275367
|
+
return this.sources?.find((source) => source.name === sourceName)?.accessFilter ?? [];
|
|
275368
|
+
}
|
|
274943
275369
|
filterGivensToModelSurface(givens) {
|
|
274944
275370
|
if (!givens)
|
|
274945
275371
|
return givens;
|
|
@@ -275025,28 +275451,7 @@ var init_model = __esm(() => {
|
|
|
275025
275451
|
structs.push(...derivedStructsReachable(structs, modelDef));
|
|
275026
275452
|
return structs;
|
|
275027
275453
|
}
|
|
275028
|
-
|
|
275029
|
-
hasAnyPartitionNote() {
|
|
275030
|
-
if (this.anyPartitionNote !== undefined)
|
|
275031
|
-
return this.anyPartitionNote;
|
|
275032
|
-
this.anyPartitionNote = (() => {
|
|
275033
|
-
const modelDef = this.modelDef;
|
|
275034
|
-
if (!modelDef)
|
|
275035
|
-
return false;
|
|
275036
|
-
try {
|
|
275037
|
-
for (const struct of this.reachableStructsForNoteSweep(modelDef)) {
|
|
275038
|
-
if (containsPartitionAnnotationTag(annotationTexts(struct.annotations) ?? [])) {
|
|
275039
|
-
return true;
|
|
275040
|
-
}
|
|
275041
|
-
}
|
|
275042
|
-
return false;
|
|
275043
|
-
} catch {
|
|
275044
|
-
return true;
|
|
275045
|
-
}
|
|
275046
|
-
})();
|
|
275047
|
-
return this.anyPartitionNote;
|
|
275048
|
-
}
|
|
275049
|
-
async assertAuthorized(sourceName, _givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
|
|
275454
|
+
async assertAuthorized(sourceName, givens, bypassAuthorize = false, graftScope = this.defaultGraftScope()) {
|
|
275050
275455
|
if (bypassAuthorize) {
|
|
275051
275456
|
this.noteAuthorizeBypass("source", sourceName);
|
|
275052
275457
|
return;
|
|
@@ -275057,10 +275462,7 @@ var init_model = __esm(() => {
|
|
|
275057
275462
|
const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
275058
275463
|
if (resolution.shape === "row_level")
|
|
275059
275464
|
continue;
|
|
275060
|
-
|
|
275061
|
-
if (resolution.cause)
|
|
275062
|
-
recordRowLevelGateRejected(resolution.cause);
|
|
275063
|
-
throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
|
|
275465
|
+
denyUnlessAdmitted(resolution, givens, entry.label);
|
|
275064
275466
|
}
|
|
275065
275467
|
return;
|
|
275066
275468
|
}
|
|
@@ -275095,7 +275497,7 @@ var init_model = __esm(() => {
|
|
|
275095
275497
|
const onDiskGates = this.entryPointGatesBySource.get(ownSourceName);
|
|
275096
275498
|
if (onDiskGates) {
|
|
275097
275499
|
const SEP = "\x1F";
|
|
275098
|
-
const keyOf = (entry) => `${entry.label}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
|
|
275500
|
+
const keyOf = (entry) => `${entry.label}${SEP}${entry.route}${SEP}${entry.exprs.join(SEP)}${SEP}${entry.selfContained}`;
|
|
275099
275501
|
const byKey = new Map(entryPointGates.map((entry) => [keyOf(entry), entry]));
|
|
275100
275502
|
for (const entry of onDiskGates) {
|
|
275101
275503
|
byKey.set(keyOf(entry), entry);
|
|
@@ -275115,7 +275517,7 @@ var init_model = __esm(() => {
|
|
|
275115
275517
|
return this.rowLevelFilteredRunnables.has(runnable);
|
|
275116
275518
|
}
|
|
275117
275519
|
async probeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate = false) {
|
|
275118
|
-
const { entryPointGates, modelDef
|
|
275520
|
+
const { entryPointGates, modelDef } = await this.collectAuthorizeEntryPointGates(runnable, givens, graftScope, skipOwnSourceGate);
|
|
275119
275521
|
const rowLevel = [];
|
|
275120
275522
|
for (const entry of entryPointGates) {
|
|
275121
275523
|
const resolution = modelDef ? await this.resolveGateShape(entry, modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
@@ -275129,20 +275531,7 @@ var init_model = __esm(() => {
|
|
|
275129
275531
|
});
|
|
275130
275532
|
continue;
|
|
275131
275533
|
}
|
|
275132
|
-
|
|
275133
|
-
if (resolution.cause)
|
|
275134
|
-
recordRowLevelGateRejected(resolution.cause);
|
|
275135
|
-
throw new AccessDeniedError(`Access denied for source "${entry.label}".`);
|
|
275136
|
-
}
|
|
275137
|
-
try {
|
|
275138
|
-
rowLevel.push(...await resolvePartitionGraftEntries(struct, modelDef, graftScope, this.gateClassificationDeps()));
|
|
275139
|
-
} catch (err) {
|
|
275140
|
-
recordRowLevelGateDecision("denied_by_gate");
|
|
275141
|
-
logger.debug("Partition filter could not be resolved; denying", {
|
|
275142
|
-
modelPath: this.modelPath,
|
|
275143
|
-
error: err instanceof Error ? err.message : String(err)
|
|
275144
|
-
});
|
|
275145
|
-
throw new AccessDeniedError(`Access denied for source "${struct?.as ?? struct?.name ?? "unknown"}".`);
|
|
275534
|
+
denyUnlessAdmitted(resolution, givens, entry.label);
|
|
275146
275535
|
}
|
|
275147
275536
|
return rowLevel;
|
|
275148
275537
|
}
|
|
@@ -275156,12 +275545,12 @@ var init_model = __esm(() => {
|
|
|
275156
275545
|
if (compositeResolvedSourceDef) {
|
|
275157
275546
|
gates.push(...this.collectEntryPointGates(compositeResolvedSourceDef, modelDef, seen, true, undefined, struct ? ownLevelNotes(struct.annotations) : []));
|
|
275158
275547
|
}
|
|
275159
|
-
return gates.length > 0
|
|
275548
|
+
return gates.length > 0;
|
|
275160
275549
|
} catch {
|
|
275161
275550
|
return true;
|
|
275162
275551
|
}
|
|
275163
275552
|
}
|
|
275164
|
-
async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query) {
|
|
275553
|
+
async assertRequestDeclaredEntryPointIsNotLaundered(runnable, query, givens, graftScope) {
|
|
275165
275554
|
if (!this.declaresAnyGate())
|
|
275166
275555
|
return;
|
|
275167
275556
|
const entryPoint = await this.resolveAuthorizeSourceFromRunnable(runnable);
|
|
@@ -275170,15 +275559,51 @@ var init_model = __esm(() => {
|
|
|
275170
275559
|
if (this.entryPointGatesBySource.has(entryPoint))
|
|
275171
275560
|
return;
|
|
275172
275561
|
const proof = this.requestChainProvesUngated(entryPoint, buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query)));
|
|
275173
|
-
if (proof.proven)
|
|
275562
|
+
if (proof.proven) {
|
|
275563
|
+
for (const base of proof.gatedBases) {
|
|
275564
|
+
for (const entry of this.entryPointGatesBySource.get(base) ?? []) {
|
|
275565
|
+
const resolution = this.modelDef ? await this.resolveGateShape(entry, this.modelDef, graftScope) : { shape: "rejected", cause: undefined };
|
|
275566
|
+
if (resolution.shape === "row_level") {
|
|
275567
|
+
this.denyLaunderedEntryPoint(entryPoint, "row_filter_not_graftable", base);
|
|
275568
|
+
}
|
|
275569
|
+
denyUnlessAdmitted(resolution, givens, entryPoint);
|
|
275570
|
+
}
|
|
275571
|
+
}
|
|
275572
|
+
return;
|
|
275573
|
+
}
|
|
275574
|
+
this.denyLaunderedEntryPoint(entryPoint, proof.reason, proof.at);
|
|
275575
|
+
}
|
|
275576
|
+
async assertLocksOnRequestDeclaredBases(entryPoint, query, givens, bypassAuthorize) {
|
|
275577
|
+
if (!this.declaresAnyGate())
|
|
275174
275578
|
return;
|
|
275579
|
+
const basesOf = buildDerivationBaseMap(stripMalloyCommentsAndLiterals(query));
|
|
275580
|
+
const seen = new Set;
|
|
275581
|
+
const worklist = entryPoint ? [entryPoint] : [...basesOf.keys()];
|
|
275582
|
+
for (let i = 0;i < worklist.length; i++) {
|
|
275583
|
+
const name = worklist[i];
|
|
275584
|
+
if (seen.has(name))
|
|
275585
|
+
continue;
|
|
275586
|
+
seen.add(name);
|
|
275587
|
+
if (seen.size > REQUEST_CHAIN_MAX_NAMES)
|
|
275588
|
+
return;
|
|
275589
|
+
if (this.entryPointGatesBySource.has(name)) {
|
|
275590
|
+
try {
|
|
275591
|
+
await this.assertAuthorized(name, givens, bypassAuthorize);
|
|
275592
|
+
} catch (error) {
|
|
275593
|
+
if (error instanceof AccessDeniedError) {
|
|
275594
|
+
this.assertQueryBoundaryEarly(name, undefined, undefined);
|
|
275595
|
+
}
|
|
275596
|
+
throw error;
|
|
275597
|
+
}
|
|
275598
|
+
continue;
|
|
275599
|
+
}
|
|
275600
|
+
for (const base of basesOf.get(name) ?? [])
|
|
275601
|
+
worklist.push(base);
|
|
275602
|
+
}
|
|
275603
|
+
}
|
|
275604
|
+
denyLaunderedEntryPoint(entryPoint, reason, at) {
|
|
275175
275605
|
recordRowLevelGateDecision("denied_by_gate");
|
|
275176
|
-
logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", {
|
|
275177
|
-
modelPath: this.modelPath,
|
|
275178
|
-
entryPoint,
|
|
275179
|
-
reason: proof.reason,
|
|
275180
|
-
at: proof.at
|
|
275181
|
-
});
|
|
275606
|
+
logger.debug("Request-declared entry point in a gated model is not provably ungated; denying", { modelPath: this.modelPath, entryPoint, reason, at });
|
|
275182
275607
|
throw new AccessDeniedError(`Access denied for source "${entryPoint}".`);
|
|
275183
275608
|
}
|
|
275184
275609
|
declaresAnyGateMemo;
|
|
@@ -275188,6 +275613,7 @@ var init_model = __esm(() => {
|
|
|
275188
275613
|
}
|
|
275189
275614
|
requestChainProvesUngated(entryPoint, basesOf) {
|
|
275190
275615
|
const seen = new Set;
|
|
275616
|
+
const gatedBases = [];
|
|
275191
275617
|
const worklist = [entryPoint];
|
|
275192
275618
|
for (let i = 0;i < worklist.length; i++) {
|
|
275193
275619
|
const name = worklist[i];
|
|
@@ -275199,13 +275625,8 @@ var init_model = __esm(() => {
|
|
|
275199
275625
|
}
|
|
275200
275626
|
const modelGates = this.entryPointGatesBySource.get(name);
|
|
275201
275627
|
if (modelGates !== undefined) {
|
|
275202
|
-
if (modelGates.length > 0)
|
|
275203
|
-
|
|
275204
|
-
proven: false,
|
|
275205
|
-
reason: "reaches_gated_source",
|
|
275206
|
-
at: name
|
|
275207
|
-
};
|
|
275208
|
-
}
|
|
275628
|
+
if (modelGates.length > 0)
|
|
275629
|
+
gatedBases.push(name);
|
|
275209
275630
|
continue;
|
|
275210
275631
|
}
|
|
275211
275632
|
const bases = basesOf.get(name);
|
|
@@ -275215,7 +275636,7 @@ var init_model = __esm(() => {
|
|
|
275215
275636
|
for (const base of bases)
|
|
275216
275637
|
worklist.push(base);
|
|
275217
275638
|
}
|
|
275218
|
-
return { proven: true };
|
|
275639
|
+
return { proven: true, gatedBases };
|
|
275219
275640
|
}
|
|
275220
275641
|
async authorizeAndBindRunnable(runnable, givens, options) {
|
|
275221
275642
|
if (options?.bypassAuthorize) {
|
|
@@ -275226,7 +275647,7 @@ var init_model = __esm(() => {
|
|
|
275226
275647
|
const rowLevel = await this.probeEntryPointGates(runnable, givens, graftScope, options?.skipOwnSourceGate ?? false);
|
|
275227
275648
|
if (rowLevel.length === 0) {
|
|
275228
275649
|
if (options?.callerQueryText && !options?.skipOwnSourceGate) {
|
|
275229
|
-
await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText);
|
|
275650
|
+
await this.assertRequestDeclaredEntryPointIsNotLaundered(runnable, options.callerQueryText, givens, graftScope);
|
|
275230
275651
|
}
|
|
275231
275652
|
return runnable;
|
|
275232
275653
|
}
|
|
@@ -275291,7 +275712,7 @@ var init_model = __esm(() => {
|
|
|
275291
275712
|
}
|
|
275292
275713
|
async resolveGateShape(entry, originModelDef, graftScope) {
|
|
275293
275714
|
const result = await resolveGateShape(entry, originModelDef, graftScope, this.gateClassificationDeps());
|
|
275294
|
-
if (result.shape
|
|
275715
|
+
if (result.shape !== "rejected") {
|
|
275295
275716
|
for (const name of result.givenNames) {
|
|
275296
275717
|
this.authorizeReferencedGivenNames.add(name);
|
|
275297
275718
|
}
|
|
@@ -275432,7 +275853,11 @@ var init_model = __esm(() => {
|
|
|
275432
275853
|
return false;
|
|
275433
275854
|
}
|
|
275434
275855
|
async assertAuthorizedForText(text, givens) {
|
|
275435
|
-
|
|
275856
|
+
const target = extractRunTargetSourceName(text);
|
|
275857
|
+
await this.assertAuthorized(target, givens);
|
|
275858
|
+
if (!hasCallerAuthorizeAnnotation(text)) {
|
|
275859
|
+
await this.assertLocksOnRequestDeclaredBases(target ?? "", text, givens);
|
|
275860
|
+
}
|
|
275436
275861
|
}
|
|
275437
275862
|
async assertAuthorizedForRunnable(runnable, givens) {
|
|
275438
275863
|
await this.assertAuthorizedForAllSources(runnable, givens, false, {
|
|
@@ -275490,7 +275915,7 @@ var init_model = __esm(() => {
|
|
|
275490
275915
|
filterMap = sourceResult.filterMap;
|
|
275491
275916
|
const queryResult = Model.getQueries(modelDef);
|
|
275492
275917
|
queries = queryResult.queries;
|
|
275493
|
-
|
|
275918
|
+
assertNoRetiredRouteMarkers(collectRetiredRouteMarkers(modelDef));
|
|
275494
275919
|
assertNoMisplacedAuthorizeAnnotations([
|
|
275495
275920
|
...sourceResult.misplacedAuthorize,
|
|
275496
275921
|
...queryResult.misplacedAuthorize
|
|
@@ -275498,20 +275923,20 @@ var init_model = __esm(() => {
|
|
|
275498
275923
|
const legacyStringGates = findLegacyStringGates(sourceResult.authorizeOwnNotes);
|
|
275499
275924
|
legacyStringGates.forEach(() => recordRowLevelGateRejected("legacy_string_gate"));
|
|
275500
275925
|
assertNoLegacyStringGate(legacyStringGates);
|
|
275501
|
-
|
|
275926
|
+
assertAuthorizeGrammarValid(modelDef, sourceResult.authorizeMap, sourceResult.authorizeOwnNotes, computeGivenDeclaredTypes(givens), (_sourceName, route) => recordAuthorizeAdmitAllGate(route), sourceResult.attributedAuthorizeOwnNotes);
|
|
275502
275927
|
const compiledModelDef = modelDef;
|
|
275503
275928
|
await validateAuthorizeProbes(modelMaterializer, {
|
|
275504
275929
|
authorizeMap: sourceResult.authorizeMap,
|
|
275505
275930
|
authorizeOwnNotes: sourceResult.attributedAuthorizeOwnNotes,
|
|
275506
275931
|
onRowLevelGateRejected: recordRowLevelGateRejected,
|
|
275507
|
-
onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(
|
|
275508
|
-
onOwnRowLevelConditionCompiled: (sourceName, condition) => {
|
|
275932
|
+
onRowLevelGateUnexpressible: (sourceName, detail) => logger.warn("Row-level #(access_filter) gate not expressible at this entry point; every query against it will be denied", { packageName, modelPath, sourceName, detail }),
|
|
275933
|
+
onOwnRowLevelConditionCompiled: (sourceName, condition, route) => {
|
|
275509
275934
|
const struct = compiledModelDef.contents[sourceName];
|
|
275510
275935
|
if (!struct || !isSourceDef7(struct))
|
|
275511
275936
|
return;
|
|
275512
|
-
validateSourceLineGateGivenUsage(sourceName, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
|
|
275937
|
+
validateSourceLineGateGivenUsage(sourceName, route, struct, condition.refSummary, condition.e, compiledModelDef, (cause, detail) => {
|
|
275513
275938
|
recordRowLevelGateRejected(cause);
|
|
275514
|
-
logger.warn(
|
|
275939
|
+
logger.warn(`#(${route}) gate warning`, {
|
|
275515
275940
|
packageName,
|
|
275516
275941
|
modelPath,
|
|
275517
275942
|
sourceName,
|
|
@@ -275647,7 +276072,12 @@ var init_model = __esm(() => {
|
|
|
275647
276072
|
getDashboardModelFacts() {
|
|
275648
276073
|
if (!this.modelDef)
|
|
275649
276074
|
return;
|
|
275650
|
-
return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [
|
|
276075
|
+
return readDashboardModelFacts(this.modelPath, this.modelDef, (this.givens ?? []).map((given) => given.name).filter((name) => name !== undefined), new Map((this.sources ?? []).flatMap((source) => source.name ? [
|
|
276076
|
+
[
|
|
276077
|
+
source.name,
|
|
276078
|
+
gateGivenSource(this.sources ?? [], source.name) ?? []
|
|
276079
|
+
]
|
|
276080
|
+
] : [])));
|
|
275651
276081
|
}
|
|
275652
276082
|
hasEmptyDiscoverySurface() {
|
|
275653
276083
|
if (!this.discoveryCurationEnabled)
|
|
@@ -276127,6 +276557,9 @@ var init_model = __esm(() => {
|
|
|
276127
276557
|
const earlySource = sourceName || (queryName ? this.queries?.find((q) => q.name === queryName)?.sourceName : undefined) || (surfaceName && !this.sources?.some((s) => s.name === surfaceName) ? this.queries?.find((q) => q.name === surfaceName)?.sourceName : undefined) || surfaceName;
|
|
276128
276558
|
if (earlySource) {
|
|
276129
276559
|
await this.assertAuthorized(earlySource, givens ?? {}, bypassAuthorize);
|
|
276560
|
+
if (query && !hasCallerAuthorizeAnnotation(query) && !this.entryPointGatesBySource.has(earlySource)) {
|
|
276561
|
+
await this.assertLocksOnRequestDeclaredBases(earlySource, query, givens ?? {}, bypassAuthorize);
|
|
276562
|
+
}
|
|
276130
276563
|
}
|
|
276131
276564
|
try {
|
|
276132
276565
|
for (const [field, callerText] of [
|
|
@@ -276175,7 +276608,7 @@ run: ${sourceName ? `${quoteMalloyIdentifier2(sourceName)} -> ` : ""}${quoteMall
|
|
|
276175
276608
|
liveRunnable = runnable;
|
|
276176
276609
|
const storageRoutingPossible = getPersistStorageMode() === "on" && this.serveBindings.length > 0 && !!this.serveDestinationConfig;
|
|
276177
276610
|
const preaggServeMaterializer = this.preaggregateServeMaterializer;
|
|
276178
|
-
const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize &&
|
|
276611
|
+
const routingBlockedByRowLevelGate = (storageRoutingPossible || !!preaggServeMaterializer) && !bypassAuthorize && this.hasAnyAuthorizeNote() && await this.queryEntryPointHasRowLevelGate(runnable);
|
|
276179
276612
|
if (routingBlockedByRowLevelGate) {
|
|
276180
276613
|
recordStorageServeRouting("blocked_by_row_level_gate");
|
|
276181
276614
|
}
|
|
@@ -277659,12 +278092,12 @@ var require_recursive_readdir = __commonJS((exports, module) => {
|
|
|
277659
278092
|
ignores = [];
|
|
277660
278093
|
}
|
|
277661
278094
|
if (!callback) {
|
|
277662
|
-
return new Promise(function(
|
|
278095
|
+
return new Promise(function(resolve5, reject2) {
|
|
277663
278096
|
readdir3(path11, ignores || [], function(err, data) {
|
|
277664
278097
|
if (err) {
|
|
277665
278098
|
reject2(err);
|
|
277666
278099
|
} else {
|
|
277667
|
-
|
|
278100
|
+
resolve5(data);
|
|
277668
278101
|
}
|
|
277669
278102
|
});
|
|
277670
278103
|
});
|
|
@@ -284918,7 +285351,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
284918
285351
|
target.fragment = relative5.fragment;
|
|
284919
285352
|
return target;
|
|
284920
285353
|
}
|
|
284921
|
-
function
|
|
285354
|
+
function resolve6(baseURI, relativeURI, options) {
|
|
284922
285355
|
var schemelessOptions = assign({ scheme: "null" }, options);
|
|
284923
285356
|
return serialize(resolveComponents(parse(baseURI, schemelessOptions), parse(relativeURI, schemelessOptions), schemelessOptions, true), schemelessOptions);
|
|
284924
285357
|
}
|
|
@@ -285186,7 +285619,7 @@ var require_uri_all = __commonJS((exports, module) => {
|
|
|
285186
285619
|
exports2.removeDotSegments = removeDotSegments;
|
|
285187
285620
|
exports2.serialize = serialize;
|
|
285188
285621
|
exports2.resolveComponents = resolveComponents;
|
|
285189
|
-
exports2.resolve =
|
|
285622
|
+
exports2.resolve = resolve6;
|
|
285190
285623
|
exports2.normalize = normalize2;
|
|
285191
285624
|
exports2.equal = equal;
|
|
285192
285625
|
exports2.escapeComponent = escapeComponent;
|
|
@@ -285552,20 +285985,20 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
285552
285985
|
var util5 = require_util12();
|
|
285553
285986
|
var SchemaObject = require_schema_obj();
|
|
285554
285987
|
var traverse = require_json_schema_traverse();
|
|
285555
|
-
module.exports =
|
|
285556
|
-
|
|
285557
|
-
|
|
285558
|
-
|
|
285559
|
-
|
|
285560
|
-
|
|
285561
|
-
|
|
285562
|
-
function
|
|
285988
|
+
module.exports = resolve6;
|
|
285989
|
+
resolve6.normalizeId = normalizeId;
|
|
285990
|
+
resolve6.fullPath = getFullPath;
|
|
285991
|
+
resolve6.url = resolveUrl;
|
|
285992
|
+
resolve6.ids = resolveIds;
|
|
285993
|
+
resolve6.inlineRef = inlineRef;
|
|
285994
|
+
resolve6.schema = resolveSchema;
|
|
285995
|
+
function resolve6(compile, root, ref) {
|
|
285563
285996
|
var refVal = this._refs[ref];
|
|
285564
285997
|
if (typeof refVal == "string") {
|
|
285565
285998
|
if (this._refs[refVal])
|
|
285566
285999
|
refVal = this._refs[refVal];
|
|
285567
286000
|
else
|
|
285568
|
-
return
|
|
286001
|
+
return resolve6.call(this, compile, root, refVal);
|
|
285569
286002
|
}
|
|
285570
286003
|
refVal = refVal || this._schemas[ref];
|
|
285571
286004
|
if (refVal instanceof SchemaObject) {
|
|
@@ -285788,7 +286221,7 @@ var require_resolve2 = __commonJS((exports, module) => {
|
|
|
285788
286221
|
|
|
285789
286222
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/error_classes.js
|
|
285790
286223
|
var require_error_classes = __commonJS((exports, module) => {
|
|
285791
|
-
var
|
|
286224
|
+
var resolve6 = require_resolve2();
|
|
285792
286225
|
module.exports = {
|
|
285793
286226
|
Validation: errorSubclass(ValidationError),
|
|
285794
286227
|
MissingRef: errorSubclass(MissingRefError)
|
|
@@ -285803,8 +286236,8 @@ var require_error_classes = __commonJS((exports, module) => {
|
|
|
285803
286236
|
};
|
|
285804
286237
|
function MissingRefError(baseId, ref, message) {
|
|
285805
286238
|
this.message = message || MissingRefError.message(baseId, ref);
|
|
285806
|
-
this.missingRef =
|
|
285807
|
-
this.missingSchema =
|
|
286239
|
+
this.missingRef = resolve6.url(baseId, ref);
|
|
286240
|
+
this.missingSchema = resolve6.normalizeId(resolve6.fullPath(this.missingRef));
|
|
285808
286241
|
}
|
|
285809
286242
|
function errorSubclass(Subclass) {
|
|
285810
286243
|
Subclass.prototype = Object.create(Error.prototype);
|
|
@@ -286346,7 +286779,7 @@ var require_validate4 = __commonJS((exports, module) => {
|
|
|
286346
286779
|
|
|
286347
286780
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/compile/index.js
|
|
286348
286781
|
var require_compile2 = __commonJS((exports, module) => {
|
|
286349
|
-
var
|
|
286782
|
+
var resolve6 = require_resolve2();
|
|
286350
286783
|
var util5 = require_util12();
|
|
286351
286784
|
var errorClasses = require_error_classes();
|
|
286352
286785
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -286406,7 +286839,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286406
286839
|
RULES,
|
|
286407
286840
|
validate: validateGenerator,
|
|
286408
286841
|
util: util5,
|
|
286409
|
-
resolve:
|
|
286842
|
+
resolve: resolve6,
|
|
286410
286843
|
resolveRef,
|
|
286411
286844
|
usePattern,
|
|
286412
286845
|
useDefault,
|
|
@@ -286445,7 +286878,7 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286445
286878
|
return validate;
|
|
286446
286879
|
}
|
|
286447
286880
|
function resolveRef(baseId2, ref, isRoot) {
|
|
286448
|
-
ref =
|
|
286881
|
+
ref = resolve6.url(baseId2, ref);
|
|
286449
286882
|
var refIndex = refs[ref];
|
|
286450
286883
|
var _refVal, refCode;
|
|
286451
286884
|
if (refIndex !== undefined) {
|
|
@@ -286462,11 +286895,11 @@ var require_compile2 = __commonJS((exports, module) => {
|
|
|
286462
286895
|
}
|
|
286463
286896
|
}
|
|
286464
286897
|
refCode = addLocalRef(ref);
|
|
286465
|
-
var v2 =
|
|
286898
|
+
var v2 = resolve6.call(self2, localCompile, root, ref);
|
|
286466
286899
|
if (v2 === undefined) {
|
|
286467
286900
|
var localSchema = localRefs && localRefs[ref];
|
|
286468
286901
|
if (localSchema) {
|
|
286469
|
-
v2 =
|
|
286902
|
+
v2 = resolve6.inlineRef(localSchema, opts.inlineRefs) ? localSchema : compile.call(self2, localSchema, root, localRefs, baseId2);
|
|
286470
286903
|
}
|
|
286471
286904
|
}
|
|
286472
286905
|
if (v2 === undefined) {
|
|
@@ -289983,7 +290416,7 @@ var require_data2 = __commonJS((exports, module) => {
|
|
|
289983
290416
|
// ../../node_modules/@modelcontextprotocol/sdk/node_modules/ajv/lib/ajv.js
|
|
289984
290417
|
var require_ajv = __commonJS((exports, module) => {
|
|
289985
290418
|
var compileSchema = require_compile2();
|
|
289986
|
-
var
|
|
290419
|
+
var resolve6 = require_resolve2();
|
|
289987
290420
|
var Cache = require_cache();
|
|
289988
290421
|
var SchemaObject = require_schema_obj();
|
|
289989
290422
|
var stableStringify = require_fast_json_stable_stringify();
|
|
@@ -290075,7 +290508,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290075
290508
|
var id = this._getId(schema);
|
|
290076
290509
|
if (id !== undefined && typeof id != "string")
|
|
290077
290510
|
throw new Error("schema id must be string");
|
|
290078
|
-
key =
|
|
290511
|
+
key = resolve6.normalizeId(key || id);
|
|
290079
290512
|
checkUnique(this, key);
|
|
290080
290513
|
this._schemas[key] = this._addSchema(schema, _skipValidation, _meta, true);
|
|
290081
290514
|
return this;
|
|
@@ -290121,7 +290554,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290121
290554
|
}
|
|
290122
290555
|
}
|
|
290123
290556
|
function _getSchemaFragment(self2, ref) {
|
|
290124
|
-
var res =
|
|
290557
|
+
var res = resolve6.schema.call(self2, { schema: {} }, ref);
|
|
290125
290558
|
if (res) {
|
|
290126
290559
|
var { schema, root, baseId } = res;
|
|
290127
290560
|
var v = compileSchema.call(self2, schema, root, undefined, baseId);
|
|
@@ -290137,7 +290570,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290137
290570
|
}
|
|
290138
290571
|
}
|
|
290139
290572
|
function _getSchemaObj(self2, keyRef) {
|
|
290140
|
-
keyRef =
|
|
290573
|
+
keyRef = resolve6.normalizeId(keyRef);
|
|
290141
290574
|
return self2._schemas[keyRef] || self2._refs[keyRef] || self2._fragments[keyRef];
|
|
290142
290575
|
}
|
|
290143
290576
|
function removeSchema(schemaKeyRef) {
|
|
@@ -290165,7 +290598,7 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290165
290598
|
this._cache.del(cacheKey);
|
|
290166
290599
|
var id = this._getId(schemaKeyRef);
|
|
290167
290600
|
if (id) {
|
|
290168
|
-
id =
|
|
290601
|
+
id = resolve6.normalizeId(id);
|
|
290169
290602
|
delete this._schemas[id];
|
|
290170
290603
|
delete this._refs[id];
|
|
290171
290604
|
}
|
|
@@ -290190,14 +290623,14 @@ var require_ajv = __commonJS((exports, module) => {
|
|
|
290190
290623
|
if (cached2)
|
|
290191
290624
|
return cached2;
|
|
290192
290625
|
shouldAddSchema = shouldAddSchema || this._opts.addUsedSchema !== false;
|
|
290193
|
-
var id =
|
|
290626
|
+
var id = resolve6.normalizeId(this._getId(schema));
|
|
290194
290627
|
if (id && shouldAddSchema)
|
|
290195
290628
|
checkUnique(this, id);
|
|
290196
290629
|
var willValidate = this._opts.validateSchema !== false && !skipValidation;
|
|
290197
290630
|
var recursiveMeta;
|
|
290198
|
-
if (willValidate && !(recursiveMeta = id && id ==
|
|
290631
|
+
if (willValidate && !(recursiveMeta = id && id == resolve6.normalizeId(schema.$schema)))
|
|
290199
290632
|
this.validateSchema(schema, true);
|
|
290200
|
-
var localRefs =
|
|
290633
|
+
var localRefs = resolve6.ids.call(this, schema);
|
|
290201
290634
|
var schemaObj = new SchemaObject({
|
|
290202
290635
|
id,
|
|
290203
290636
|
schema,
|
|
@@ -299430,7 +299863,7 @@ function getMalloyErrorDetails(operation, modelIdentifier, error) {
|
|
|
299430
299863
|
refined = true;
|
|
299431
299864
|
const [, sourceName] = accessDeniedMatch;
|
|
299432
299865
|
suggestions = [
|
|
299433
|
-
`Suggestion: Access to source '${sourceName}' is restricted by an
|
|
299866
|
+
`Suggestion: Access to source '${sourceName}' is restricted by an authorize gate. Supply the givens its authorize expression requires (e.g. a role/region given) and retry. This is an authorization denial, not a syntax error.`
|
|
299434
299867
|
];
|
|
299435
299868
|
} else if (viewNotFoundMatch) {
|
|
299436
299869
|
refined = true;
|
|
@@ -300544,12 +300977,15 @@ var REASON_BY_UNAVAILABLE = {
|
|
|
300544
300977
|
"too-many-entities": "too-many-entities",
|
|
300545
300978
|
error: "provider-error"
|
|
300546
300979
|
};
|
|
300547
|
-
function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false) {
|
|
300980
|
+
function toSourceResults(results, sourceContext, environmentName, packageName, searchTexts = new Map, includeCode = false, droppedSources = new Set) {
|
|
300548
300981
|
const bySource = new Map;
|
|
300549
300982
|
const cardFor = (name, modelPathFallback) => {
|
|
300550
300983
|
if (!name)
|
|
300551
300984
|
return;
|
|
300552
300985
|
const key = sourceContextKey(modelPathFallback, name);
|
|
300986
|
+
if (droppedSources.has(key)) {
|
|
300987
|
+
return;
|
|
300988
|
+
}
|
|
300553
300989
|
let entry = bySource.get(key);
|
|
300554
300990
|
if (!entry) {
|
|
300555
300991
|
const ctx = sourceContext.get(key);
|
|
@@ -300564,6 +301000,7 @@ function toSourceResults(results, sourceContext, environmentName, packageName, s
|
|
|
300564
301000
|
...ctx?.oneLineSummary ? { one_line_summary: ctx.oneLineSummary } : {},
|
|
300565
301001
|
...ctx?.doc ? { docs: ctx.doc } : {},
|
|
300566
301002
|
...ctx?.givens ? { givens: ctx.givens } : {},
|
|
301003
|
+
...ctx?.accessFilter ? { accessFilter: ctx.accessFilter } : {},
|
|
300567
301004
|
...ctx?.authorize ? { authorize: ctx.authorize } : {},
|
|
300568
301005
|
...ctx?.filters ? { filter_params: ctx.filters } : {},
|
|
300569
301006
|
joins: ctx?.joins ?? []
|
|
@@ -300684,7 +301121,7 @@ function projectEntity(e, environmentName, packageName) {
|
|
|
300684
301121
|
}
|
|
300685
301122
|
function finishRanked(args) {
|
|
300686
301123
|
const windowed = windowBySource(args.rows, args.max);
|
|
300687
|
-
const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode);
|
|
301124
|
+
const sources = toSourceResults(windowed.rows, args.sourceContext, args.environmentName, args.packageName, args.searchTexts, args.includeCode, args.droppedSources);
|
|
300688
301125
|
return {
|
|
300689
301126
|
sources,
|
|
300690
301127
|
totalSources: windowed.totalSources,
|
|
@@ -300931,10 +301368,15 @@ function collectJoinedFields(args) {
|
|
|
300931
301368
|
});
|
|
300932
301369
|
}
|
|
300933
301370
|
}
|
|
301371
|
+
function isUnconditionalDenyAuthorize(apiSource) {
|
|
301372
|
+
const isDeny = (expr) => expr.trim().toLowerCase() === "false";
|
|
301373
|
+
return (apiSource.accessFilter ?? []).some(isDeny) || (apiSource.authorize ?? []).some(isDeny);
|
|
301374
|
+
}
|
|
300934
301375
|
async function collectEntities(pkg) {
|
|
300935
301376
|
const models = [...await pkg.listModels()].sort((a, b) => comparePaths(a.path ?? "", b.path ?? ""));
|
|
300936
301377
|
const entities = [];
|
|
300937
301378
|
const governance = new Map;
|
|
301379
|
+
const droppedSources = new Set;
|
|
300938
301380
|
const sourceTextFor = makeSourceTextReader(pkg);
|
|
300939
301381
|
let n = 0;
|
|
300940
301382
|
for (const apiModel of models) {
|
|
@@ -300948,10 +301390,18 @@ async function collectEntities(pkg) {
|
|
|
300948
301390
|
const queries = model.getQueries() ?? [];
|
|
300949
301391
|
const apiSources = model.getSources?.() ?? [];
|
|
300950
301392
|
const modelDef = model.getModelDef?.();
|
|
301393
|
+
for (const apiSource of apiSources) {
|
|
301394
|
+
if (apiSource.name && isUnconditionalDenyAuthorize(apiSource)) {
|
|
301395
|
+
droppedSources.add(sourceContextKey(modelPath, apiSource.name));
|
|
301396
|
+
}
|
|
301397
|
+
}
|
|
300951
301398
|
for (const sourceInfo of sourceInfos) {
|
|
300952
301399
|
const sourceName = sourceInfo.name;
|
|
300953
|
-
const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
|
|
300954
301400
|
const governanceKey = sourceContextKey(modelPath, sourceName);
|
|
301401
|
+
if (droppedSources.has(governanceKey)) {
|
|
301402
|
+
continue;
|
|
301403
|
+
}
|
|
301404
|
+
const provenance = readFieldProvenance(modelDef, sourceName, sourceTextFor);
|
|
300955
301405
|
if (!governance.has(governanceKey)) {
|
|
300956
301406
|
const apiSource = apiSources.find((c) => c.name === sourceName);
|
|
300957
301407
|
if (apiSource) {
|
|
@@ -300964,6 +301414,10 @@ async function collectEntities(pkg) {
|
|
|
300964
301414
|
...given.default !== undefined ? { default: given.default } : {}
|
|
300965
301415
|
}
|
|
300966
301416
|
] : []),
|
|
301417
|
+
accessFilter: (apiSource.accessFilter ?? []).map((expression) => ({
|
|
301418
|
+
expression,
|
|
301419
|
+
given_names: referencedGivenNames(expression)
|
|
301420
|
+
})),
|
|
300967
301421
|
authorize: (apiSource.authorize ?? []).map((expression) => ({
|
|
300968
301422
|
expression,
|
|
300969
301423
|
given_names: referencedGivenNames(expression)
|
|
@@ -301038,6 +301492,9 @@ async function collectEntities(pkg) {
|
|
|
301038
301492
|
continue;
|
|
301039
301493
|
if (!query.sourceName)
|
|
301040
301494
|
continue;
|
|
301495
|
+
if (droppedSources.has(sourceContextKey(modelPath, query.sourceName))) {
|
|
301496
|
+
continue;
|
|
301497
|
+
}
|
|
301041
301498
|
entities.push({
|
|
301042
301499
|
id: String(n++),
|
|
301043
301500
|
kind: "query",
|
|
@@ -301057,7 +301514,7 @@ async function collectEntities(pkg) {
|
|
|
301057
301514
|
seen.add(key);
|
|
301058
301515
|
return true;
|
|
301059
301516
|
});
|
|
301060
|
-
return { entities: collapseAliases(deduped), governance };
|
|
301517
|
+
return { entities: collapseAliases(deduped), governance, droppedSources };
|
|
301061
301518
|
}
|
|
301062
301519
|
function malloyType(field) {
|
|
301063
301520
|
const kind = field.type?.kind;
|
|
@@ -301160,6 +301617,7 @@ function buildSourceContext(collected) {
|
|
|
301160
301617
|
joins: [],
|
|
301161
301618
|
...summary ? { oneLineSummary: summary } : {},
|
|
301162
301619
|
...gates?.givens.length ? { givens: gates.givens } : {},
|
|
301620
|
+
...gates?.accessFilter.length ? { accessFilter: gates.accessFilter } : {},
|
|
301163
301621
|
...gates?.authorize.length ? { authorize: gates.authorize } : {},
|
|
301164
301622
|
...gates?.filters.length ? { filters: gates.filters } : {}
|
|
301165
301623
|
});
|
|
@@ -301207,7 +301665,8 @@ async function getPackageIndex(environmentStore, environmentName, packageName) {
|
|
|
301207
301665
|
byId,
|
|
301208
301666
|
index,
|
|
301209
301667
|
entityCount: entities.length,
|
|
301210
|
-
sourceContext: buildSourceContext(collected)
|
|
301668
|
+
sourceContext: buildSourceContext(collected),
|
|
301669
|
+
droppedSources: collected.droppedSources
|
|
301211
301670
|
};
|
|
301212
301671
|
indexCache.set(pkg, built);
|
|
301213
301672
|
logger.debug("[MCP Tool getContext] Built and cached entity index", {
|
|
@@ -301225,13 +301684,13 @@ var GET_CONTEXT_DESCRIPTION = `Retrieve the entities in a Malloy package most re
|
|
|
301225
301684
|
- Read warnings and any error/stale field before trusting a number.
|
|
301226
301685
|
- A source's joins list is complete: empty means it declares none, so write that relationship inline.
|
|
301227
301686
|
- Read a source's doc before querying: it carries grain and population rules its fields do not.
|
|
301228
|
-
- authorize
|
|
301687
|
+
- accessFilter/authorize mean gated; a deny-all source never appears here.
|
|
301229
301688
|
|
|
301230
301689
|
## Parameters
|
|
301231
301690
|
search_targets: one per concept, {target_type, search_text}; target_type is source|dimension|measure|view|join|dimensional_value, omitting search_text enumerates that type. scopes: {environment, package} + optional model_path, source, entity_name. limit caps sources (max 150, counted as cards). offset pages a listing. user_prompt: the question asked. include_code or a pinned entity_name returns code.
|
|
301232
301691
|
|
|
301233
301692
|
## Response
|
|
301234
|
-
sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, authorize
|
|
301693
|
+
sources[], best first; a source repeats once per model_path resolving it, query any. source_info: resource_id (environment/package/model_path/source) -> execute_query's environmentName/packageName/modelPath/sourceName; docs (… = truncated), one_line_summary, complete joins, givens, accessFilter/authorize, filter_params. entities[] nest under it: name, entity_type, description, data_type, relationship (fan-out), join_path, aliases, relevance, entity_id. A joined field's name IS its dotted path; use it verbatim.
|
|
301235
301694
|
ranking, returned of total_available sources, next_offset on a listing, warnings[].
|
|
301236
301695
|
Semantic fills relevance: no sources = nothing cleared the floor; below_cutoff_count of total_entities rejected. "lexical" adds retrieval_reason; only "indexing" is worth a retry.
|
|
301237
301696
|
|
|
@@ -301266,7 +301725,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301266
301725
|
});
|
|
301267
301726
|
return contextError(buildMalloyUri({ environment: environmentName, package: packageName }, "get-context"), `${environmentName}/${packageName}`, error);
|
|
301268
301727
|
}
|
|
301269
|
-
const { byId, index, sourceContext } = pkgIndex;
|
|
301728
|
+
const { byId, index, sourceContext, droppedSources } = pkgIndex;
|
|
301270
301729
|
const uri = buildMalloyUri({ environment: environmentName, package: packageName }, "get-context");
|
|
301271
301730
|
let staleNote;
|
|
301272
301731
|
try {
|
|
@@ -301304,7 +301763,7 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301304
301763
|
});
|
|
301305
301764
|
const spendable = (rows) => sourceName ? rows.filter((e) => e.kind !== "source").length : rows.length;
|
|
301306
301765
|
const cappable = spendable(inScope);
|
|
301307
|
-
const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode);
|
|
301766
|
+
const sources2 = toSourceResults(capped.map((e) => projectEntity(e, environmentName, packageName)), sourceContext, environmentName, packageName, new Map, request.includeCode, droppedSources);
|
|
301308
301767
|
const consumed = request.offset + sources2.length;
|
|
301309
301768
|
const cardsInScope = sourceName ? new Set(inScope.map((e) => sourceContextKey(e.modelPath, e.source ?? ""))).size : inScope.length;
|
|
301310
301769
|
const listingEnvelope = {
|
|
@@ -301420,7 +301879,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301420
301879
|
environmentName,
|
|
301421
301880
|
packageName,
|
|
301422
301881
|
searchTexts: searchTextsByIndex,
|
|
301423
|
-
includeCode: request.includeCode
|
|
301882
|
+
includeCode: request.includeCode,
|
|
301883
|
+
droppedSources
|
|
301424
301884
|
});
|
|
301425
301885
|
return jsonResource(uri, {
|
|
301426
301886
|
sources: sources2,
|
|
@@ -301481,7 +301941,8 @@ async function runContextQuery(request, environmentStore, extraWarnings = []) {
|
|
|
301481
301941
|
environmentName,
|
|
301482
301942
|
packageName,
|
|
301483
301943
|
searchTexts: searchTextsByIndex,
|
|
301484
|
-
includeCode: request.includeCode
|
|
301944
|
+
includeCode: request.includeCode,
|
|
301945
|
+
droppedSources
|
|
301485
301946
|
});
|
|
301486
301947
|
const envelope = {
|
|
301487
301948
|
sources,
|
|
@@ -313578,9 +314039,9 @@ class EnvironmentStore {
|
|
|
313578
314039
|
}
|
|
313579
314040
|
const file = fs9.createWriteStream(zipFilePath);
|
|
313580
314041
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
313581
|
-
await new Promise((
|
|
314042
|
+
await new Promise((resolve6, reject2) => {
|
|
313582
314043
|
file.on("error", reject2);
|
|
313583
|
-
file.on("finish",
|
|
314044
|
+
file.on("finish", resolve6);
|
|
313584
314045
|
});
|
|
313585
314046
|
await this.unzipEnvironment(zipFilePath);
|
|
313586
314047
|
logger.info(`Downloaded S3 zip file ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -313618,9 +314079,9 @@ class EnvironmentStore {
|
|
|
313618
314079
|
}
|
|
313619
314080
|
const file = fs9.createWriteStream(absoluteFilePath);
|
|
313620
314081
|
item.Body.transformToWebStream().pipeTo(Writable.toWeb(file));
|
|
313621
|
-
await new Promise((
|
|
314082
|
+
await new Promise((resolve6, reject2) => {
|
|
313622
314083
|
file.on("error", reject2);
|
|
313623
|
-
file.on("finish",
|
|
314084
|
+
file.on("finish", resolve6);
|
|
313624
314085
|
});
|
|
313625
314086
|
}));
|
|
313626
314087
|
logger.info(`Downloaded S3 directory ${s3Path} to ${absoluteDirPath}`);
|
|
@@ -313640,7 +314101,7 @@ class EnvironmentStore {
|
|
|
313640
314101
|
await fs9.promises.mkdir(absoluteDirPath, { recursive: true });
|
|
313641
314102
|
const repoUrl = `https://github.com/${owner}/${repoName}`;
|
|
313642
314103
|
const reporter = new CloneProgressReporter(cloneProgressLabel(`${owner}/${repoName}`, progressContext));
|
|
313643
|
-
await new Promise((
|
|
314104
|
+
await new Promise((resolve6, reject2) => {
|
|
313644
314105
|
esm_default2({
|
|
313645
314106
|
progress: (event) => reporter.onProgress(event)
|
|
313646
314107
|
}).clone(repoUrl, absoluteDirPath, GIT_CLONE_OPTIONS, (err) => {
|
|
@@ -313655,7 +314116,7 @@ class EnvironmentStore {
|
|
|
313655
314116
|
reject2(err);
|
|
313656
314117
|
return;
|
|
313657
314118
|
}
|
|
313658
|
-
|
|
314119
|
+
resolve6();
|
|
313659
314120
|
});
|
|
313660
314121
|
});
|
|
313661
314122
|
if (!cleanPackagePath) {
|
|
@@ -314445,7 +314906,7 @@ class Protocol {
|
|
|
314445
314906
|
}
|
|
314446
314907
|
request(request, resultSchema, options) {
|
|
314447
314908
|
const { relatedRequestId, resumptionToken, onresumptiontoken } = options !== null && options !== undefined ? options : {};
|
|
314448
|
-
return new Promise((
|
|
314909
|
+
return new Promise((resolve6, reject2) => {
|
|
314449
314910
|
var _a2, _b, _c, _d, _e, _f;
|
|
314450
314911
|
if (!this._transport) {
|
|
314451
314912
|
reject2(new Error("Not connected"));
|
|
@@ -314496,7 +314957,7 @@ class Protocol {
|
|
|
314496
314957
|
}
|
|
314497
314958
|
try {
|
|
314498
314959
|
const result = resultSchema.parse(response.result);
|
|
314499
|
-
|
|
314960
|
+
resolve6(result);
|
|
314500
314961
|
} catch (error) {
|
|
314501
314962
|
reject2(error);
|
|
314502
314963
|
}
|
|
@@ -316647,17 +317108,17 @@ var compileShape = {
|
|
|
316647
317108
|
source: exports_external.string().optional().describe("The Malloy source to validate; never executed. Required at scope append/file; optional at scope package (a what-if replacement for modelPath)."),
|
|
316648
317109
|
scope: exports_external.enum(["append", "file", "package"]).optional().describe(`What source means. "append" (default): append to modelPath. "file": compile AS modelPath to validate an edit. "package": run reload's worker compiler over all .malloy/.malloynb files without serving the result; an optional source replaces modelPath.`),
|
|
316649
317110
|
includeSql: exports_external.boolean().optional().describe("When true and the source ends in a runnable query, also return the generated SQL for inspection. The query is still not run."),
|
|
316650
|
-
givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any
|
|
317111
|
+
givens: exports_external.record(exports_external.unknown()).optional().describe("Given values for the model's given: block. Also required to satisfy any authorize gate on the target model, whether or not includeSql is set.")
|
|
316651
317112
|
};
|
|
316652
317113
|
var COMPILE_DESCRIPTION = `Compile-check Malloy without running a query. Use this while authoring instead of a throwaway execute_query.
|
|
316653
317114
|
|
|
316654
317115
|
## Scopes (the scope parameter)
|
|
316655
317116
|
- "append" (default): append source to modelPath. Use for NEW definitions; existing definitions report "Cannot redefine". Positions refer to the concatenated file.
|
|
316656
317117
|
- "file": compile source AS modelPath. Use to validate an EDIT before saving; positions match the submitted file.
|
|
316657
|
-
- "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and
|
|
317118
|
+
- "package": run reload's worker compiler over all .malloy/.malloynb files without changing the served package. Optional source replaces modelPath so importers see the edit. Diagnostics may name files hidden from discovery; no rows or SQL are returned, and authorize gates still apply to caller text. A missing exact path is warned and treated as a new file. Save and call reload_package to serve a clean edit.
|
|
316658
317119
|
|
|
316659
317120
|
## Parameters
|
|
316660
|
-
- environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and
|
|
317121
|
+
- environmentName, packageName, modelPath: required. source: required at append/file, optional at package. includeSql: append/file only. givens: model givens and authorize-gate values. Caller source may not declare an access-control gate — neither #(authorize) nor #(access_filter).
|
|
316661
317122
|
|
|
316662
317123
|
## Response
|
|
316663
317124
|
{ status: "success"|"error", diagnostics: [{ severity, message, code, model, line, character, endLine, endCharacter, replacement }], sql? }. Positions are 0-based; model is the package-relative file the diagnostic points at (which can be pre-existing content, not your source). status is "error" only when an error-severity diagnostic exists; errors are also stated in a plain text block.`;
|
|
@@ -322131,7 +322592,7 @@ End with a short **Next steps**: one or two specific deeper analyses the data co
|
|
|
322131
322592
|
{
|
|
322132
322593
|
name: "malloy-analysis-pitfalls",
|
|
322133
322594
|
description: "Checks to run on a query and its numbers before a result is presented - wrong grain, fan-out, invented field names, mismatched filters, count vs distinct. The common data-analysis pitfalls.",
|
|
322134
|
-
body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats
|
|
322595
|
+
body: "# Data Analysis Pitfalls\n\nWatch for these common mistakes throughout the analysis workflow. When you encounter one, fix it before presenting results.\n\n> **Tool names** are written bare here - `get_context`, `execute_query`, `search_malloy_docs`. The exact prefixed name depends on the host surface; match each against the tools you actually have.\n\n## Query Construction\n\n### Wrong grain / fan-out\nUsing dimensions or measures from a joined source that has a finer grain than the base source can silently multiply rows, inflating aggregates. For example, aggregating revenue while grouping by a line-item field may double- or triple-count totals. If your query touches fields from a joined source, compare `count(key_field)` to `count()`: if the row count is significantly higher than the distinct key count, you likely have fan-out.\n\nThat check reads in ONE direction only. `count(joined.field)` is a distinct count evaluated at the joined source's grain, so for an ordinary many-to-one join it is much SMALLER than `count()`, and that is the expected, correct result -- not missing data. Ten thousand flights joined to four hundred aircraft give `count(aircraft.tail_num)` of 400, and nothing is wrong. Do not read a low value as a coverage problem and do not abandon the answer over it. **Coverage is measured at the base grain**, by counting base rows whose joined value is null:\n\n```malloy\nrun: flights -> {\n aggregate:\n all_rows is count()\n missing is count() { where: aircraft.aircraft_models.seats is null }\n}\n```\n\nAn agent once refused a correct answer it had already computed because `count(aircraft.tail_num)` returned 36 against 17,875 flights; measured at the base grain, zero flights were missing a seat value.\n\n### Invented entity names\nNever guess field names. Use only the exact field paths defined in the model (find them with `get_context`). A plausible-sounding name that does not exist in the model will produce an error, or worse, silently reference the wrong field.\n\n### Mismatched filter values\nDimensional values are case-sensitive and format-specific. Common mismatches include case differences (\"Nike\" vs \"NIKE\" vs \"nike, inc.\"), partial matches (\"New York\" when the data has \"New York City\"), and aliased values (\"USA\" vs \"United States\"). A filter on a value that doesn't exist in the data silently returns zero rows without erroring. Always use the exact dimensional values from retrieval results, and if in doubt, run a distinct-values query on the dimension to confirm.\n\n### Filtering on the wrong field\nIf the user asks to filter by \"brand\", confirm which dimension corresponds to \"brand\" in the model. There may be multiple fields with similar names at different levels of the hierarchy.\n\n### Missing filters\nIf the user asks about \"last quarter\" but you don't apply a time filter, you are returning all-time data. Always check whether the question implies filters you have not yet applied.\n\n### Misinterpreted entities\nA field can exist in the model and still be the wrong choice. For example, using `revenue` (which may be gross) when the question asks about profit, or treating a count measure as if it were a sum. Cross-reference the field definitions and `#(doc)` descriptions in the model to confirm a field means what you think it means.\n\n### Semantic ambiguity\nField names or descriptions sometimes suggest one interpretation while the actual values tell a different story, for example, a field labeled \"annual ridership\" containing values that look like average weekday traffic, or a field named `revenue` that appears to represent net revenue in practice. When values don't match expectations, note whether the ambiguity affects the answer and call it out.\n\n### Fragile ad-hoc definitions\nWhen defining a new measure or dimension inline, common mistakes include assuming a field is numeric when it's actually a string, ignoring nulls in arithmetic (e.g., `a - b` yields null if either is null), and building logic around values that only cover a subset of the data.\n\nPay special attention to value coverage when defining a dimension that categorizes or subsets data. It's easy to capture too few values (missing categories that should be included) or too many (grouping in values that don't belong). For example, a `pick` expression that maps tier names to \"Premium\" and \"Standard\" might miss a tier that should be Premium, or a filter meant to isolate one product line might inadvertently include related but distinct products. Before relying on such a definition, run a distinct-values query on the underlying field to see the full set of values and confirm your logic handles them all correctly. This applies equally whether you define the logic as a new dimension or apply it directly as a `where` clause: the risk of incomplete value coverage is the same either way.\n\n### Hidden filters in views\nViews can have built-in `where` clauses that pre-filter the data. A view named `recent_orders` might only include the last 90 days, or `active_customers` might exclude certain statuses. If the view's definition is available, read it to understand what filters are baked in; they may conflict with what the user is asking for. When in doubt, query the base source directly and apply filters explicitly.\n\n## Result Interpretation\n\n### Implausible magnitudes\nIf a \"total\" is suspiciously small or large, question it. Common causes: missing filters (too large), over-filtering (too small), wrong unit (dollars vs. cents), or fan-out from joins (inflated).\n\n### Nulls distorting aggregations\nNull values are silently excluded from `avg()` and can make `sum()` results lower than expected. If a significant portion of a field's data is null, aggregations over that field may be misleading. When results seem off, compare total row count to a count of non-null values for the key field to gauge how much data is missing.\n\n### Confusing count vs. count distinct\n`count()` counts rows while measures defined with `count(field)` count distinct values of that field. Using a row count when you need distinct values (or vice versa) is a frequent source of inflated or deflated numbers, especially when the query touches joined sources.\n\n### Percentage of what?\nWhen computing percentages or shares, be explicit about the denominator. \"30% of revenue\" means nothing if you don't confirm what the total revenue is and whether it's filtered the same way.\n\n### Time period mismatches\nComparing metrics across different time periods without normalizing (e.g., comparing a full year to a partial quarter) produces misleading conclusions.\n\n### Extremes of a rate on a tiny denominator\n\"Which one is worst\" ranked on a ratio hands the top spot to whichever group has the fewest rows: one failure out of three is a 33% failure rate, and it will outrank a group with thousands. Before reporting an extreme of any rate, look at what is underneath it. Apply the model's documented minimum volume if it has one; if it has none, pick a floor, say which floor you picked, and print the denominator next to the rate rather than the rate alone.\n\n## Verification Signals\n\n### Parts don't sum to the whole\nIf you break down a total by category, the categories should sum to the total (or close to it, accounting for nulls). If they don't, something is wrong with the grain or filters.\n\n### A total the model tells you not to take\nThe check above assumes the parts are additive. Sometimes they are not, and the model says so: a doc noting that one event can be logged on two layers, or that a measure is non-additive, is telling you the sum across that dimension double-counts. Do not print a total row across such a dimension, and least of all one the user did not ask for. If a total is genuinely wanted, say why the model does not support it and give the per-category figures.\n\n### Row count surprises\nBefore interpreting results, check whether the row count makes sense. An unexpectedly high row count often indicates fan-out from a join. An unexpectedly low count may mean an overly restrictive filter.\n\n### Zero or empty results\nIf a query returns no rows, don't report \"there is no data.\" First verify that your filters are correct and the field names are right. The absence of results is usually a query problem, not a data problem."
|
|
322135
322596
|
},
|
|
322136
322597
|
{
|
|
322137
322598
|
name: "malloy-analysis-report",
|
|
@@ -323229,7 +323690,7 @@ A confirmed source architecture and a confirmed set of field definitions (rename
|
|
|
323229
323690
|
{
|
|
323230
323691
|
name: "malloy-gotchas-queries",
|
|
323231
323692
|
description: "Common Malloy query and view mistakes, for writing views, queries or notebooks - chart constraints, aggregate filters, joined field aliasing, method syntax, time truncation vs extraction.",
|
|
323232
|
-
body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n##
|
|
323693
|
+
body: "# Query & View Gotchas\n\n> **Read this before writing views or queries.** These patterns cause most query errors.\n\n## Charts: ONE Aggregate Per View\n\nCharts render only the **first** aggregate. Use exactly one aggregate per `# bar_chart` / `# line_chart` view.\n\n```malloy\n// WRONG: revenue is ignored\n# bar_chart\nview: x is { group_by: status, aggregate: order_count, revenue }\n// RIGHT: single aggregate\n# bar_chart\nview: x is { group_by: status, aggregate: revenue }\n```\n\nFor multiple metrics: nest separate chart views in a `# dashboard`, or use `y=['revenue','cost']` for multi-measure series.\n\n## Joined Fields in `order_by`: Must Alias First\n\n```malloy\n// WRONG: compile error\nview: x is { group_by: races.season_year, aggregate: pts, order_by: races.season_year }\n// RIGHT: alias then reference\nview: x is { group_by: yr is races.season_year, aggregate: pts, order_by: yr }\n```\n\nAny time you `group_by` a joined field, create an alias and use it in `order_by`.\n\n## `having:` vs `where:`: Aggregate Filters\n\n```malloy\n// WRONG: \"Aggregate expressions not allowed in where\"\nview: x is { group_by: cat, aggregate: n is count(), where: n > 10 }\n// RIGHT\nview: x is { group_by: cat, aggregate: n is count(), having: n > 10 }\n```\n\n- `where:` filters rows BEFORE aggregation (dimensions/raw columns)\n- `having:` filters AFTER aggregation (measures)\n\n## Aggregating Joined Fields: Method Syntax\n\n```malloy\n// WRONG: compile error: \"Join path is required for this calculation; use 'inventory_items.item_cost.sum()'\"\nmeasure: cogs is sum(inventory_items.item_cost)\n// RIGHT: method syntax\nmeasure: cogs is inventory_items.item_cost.sum()\n```\n\n`sum`, `avg`, `min`, and `max` over a dotted joined path all produce that compile error; the diagnostic message even tells you the exact fix. Don't worry about catching this in code review; the compiler does it for you.\n\n**Method syntax is for aggregates over a path. Scalar functions never take it.**\n\n```malloy\n// WRONG: \"something is missing before 'round'\"\naggregate: avg_price_r is avg(price).round(2)\naggregate: avg_price_r is price.avg().round(2)\n// WRONG: \"Cannot call function round(number, number) with source\"\naggregate: avg_price_r is avg_price.round(2)\ndimension: rounded is price.round(2)\n// RIGHT: scalar functions are always call form\naggregate: avg_price_r is round(avg(price), 2)\ndimension: rounded is round(price, 2)\n```\n\nTwo separate rules produce those errors:\n\n- **No method call chains onto the result of a function call.** `avg(price).round(2)` and `price.avg().round(2)` are both parse errors. The message names `round` without saying it is unsupported in that position, so it reads like a typo somewhere else. `.floor()` and `.ceil()` fail identically.\n- **Scalar functions have no method form.** `round`, `floor`, and `ceil` are always `round(x, 2)`, never `x.round(2)`, whether `x` is a named measure or a plain column.\n\n`price.avg()` and `inventory_items.item_cost.sum()` are correct because `avg` and `sum` are aggregate functions over a field path, which is exactly what method syntax is for.\n\n**Exception: `count(joined.field)` is correct, not a bug.** `count(joined.field)` is the **canonical Malloy idiom** for distinct-count through a join. Keep it as-is even when nearby `sum`/`avg`/`min`/`max` calls have to use method syntax. The closest method-syntax form `joined.count()` counts *rows* in the joined source (different semantics, differs from the distinct count when the joined field has duplicates within the joined table). The Malloy docs example `joined.count(field)` does NOT compile against current Malloy (error: `Expression illegal inside path.count()`); it only works for double-nested paths like `aircraft.count(aircraft_models.code)`.\n\n## `sum`/`avg` Need a Numeric Field\n\n```malloy\n// WRONG: \"Can't use type string\" - status is a string column\naggregate: avg_status is avg(status)\n// RIGHT: aggregate a numeric field; count a string one\naggregate: avg_price is avg(price), statuses is count(status)\n```\n\nCheck the field's type in the `get_context` result before aggregating it. A name that reads numeric (`order_number`, `zip`, `account_id`) is very often typed string.\n\n## A Measure Is Already an Aggregate\n\n```malloy\n// WRONG: \"Aggregate expression cannot be aggregate\" - flight_count is a measure\nrun: flights -> { aggregate: busiest is max(flight_count) }\n// RIGHT: aggregate per group, then take the max in a second stage\nrun: flights -> { group_by: carrier, aggregate: n is flight_count } -> { aggregate: busiest is max(n) }\n```\n\nThe message does not name the field. When it appears, look for a model measure you wrapped in another aggregate. If you only need the top row, skip the second stage: `order_by: n desc` with `limit: 1`.\n\n## Window Braces Bind to the Function\n\n```malloy\n// WRONG: \"`partition_by` is not supported for this kind of expression\"\ncalculate: share is sum_cumulative(n) / region_total { partition_by: region, order_by: n desc }\n// RIGHT: the brace goes directly after the window function call\ncalculate: share is sum_cumulative(n) { partition_by: region, order_by: n desc } / region_total\n```\n\nFor a within-group share, compute the denominator in `aggregate:` (`region_total is all(count(), region)`). A window's `{ partition_by }` does not apply to `all()`.\n\n## Dotted Paths Must Name a Declared Join\n\n```malloy\n// WRONG: the source declares the join as `carrier`, so this fails with\n// \"'carriers.name' is not a source or join\"\nrun: flights -> { group_by: carriers.name }\n// RIGHT: use the join name the source actually declares\nrun: flights -> { group_by: carrier.nickname }\n```\n\nA dotted path resolves only against a join declared on the source you are running. Confirm both the join name and the field under it in a `get_context` result; do not infer either from a table name or a plural/singular guess.\n\n## `order_by:` Can Only Name an Output Column\n\n```malloy\n// WRONG: \"Unknown field total in output space\" - total is never emitted\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: total }\n// RIGHT: order by a column the query actually outputs\nrun: orders -> { group_by: state, aggregate: revenue is sum(total), order_by: revenue }\n```\n\n`order_by:` resolves against the query's *output* columns, not the source's fields. To order by something, `group_by` or `aggregate` it first - and if it comes through a join, alias it (see above).\n\n## Chart Annotation Placement\n\nPlace `# bar_chart` / `# line_chart` on the **nested view definition**, not on `nest:` itself. Putting it on `nest:` causes \"not a repeated record\" errors.\n\n## DRY: Define in Source, Reference in View\n\n```malloy\n// WRONG: inline in view\nview: summary is { aggregate: revenue is sum(total) }\n// RIGHT: reference existing measure\nview: summary is { aggregate: revenue }\n```\n\n## Time Truncation vs Extraction\n\n| Syntax | What it does | Returns |\n|--------|--------------|---------|\n| `ts.month` | Truncates to start of month | Timestamp (`@2024-03-01`) |\n| `month(ts)` | Extracts month number | Integer (1-12) |\n| `ts.year` | Truncates to start of year | Timestamp (`@2024-01-01`) |\n| `year(ts)` | Extracts year number | Integer (2024) |\n\nUse `.month` for time series charts (proper date ordering). Use `month()` for cross-year comparison.\n\n**Year integers render with commas.** `year(ts)` displays as `2,018`. Tag with `# number=id` to suppress commas. Same for zip codes, IDs.\n\n## `?` Alternation: Use Commas to Combine Filters\n\nThe `?` operator is Malloy's **alternation operator**: a shorthand for \"match any of these values.\" `party ? 'Democrat' | 'Republican'` means `party = 'Democrat' OR party = 'Republican'`. The `|` separates the alternatives.\n\nWhen combining an alternation filter with other filters, **use a comma**:\n\n```malloy\n// CANONICAL: commas separate independent filter conditions\nwhere: is_us = true, party ? 'Democrat' | 'Republican'\n```\n\n`and` works in some arrangements (when the alternation is the second operand) but produces a confusing `'logical operator' Can't use type string` compile error when the alternation comes first. The comma form is unambiguous in every position, so just use it.\n\n## Strings: Apostrophes and Concatenation\n\n```malloy\n// WRONG: \"no viable alternative at input 's'\" - the apostrophe ends the literal\nwhere: name = 'Mac's Diner'\n// RIGHT\nwhere: name = \"Mac's Diner\"\n\n// WRONG: \"unexpected '+'\" / \"no viable alternative at input '||'\"\ngroup_by: route is origin ++ '-' ++ destination\n// RIGHT: there is no concatenation operator\ngroup_by: route is concat(origin, '-', destination)\n```\n\n## A Semicolon Inside a Clause Ends It\n\nBetween clauses a newline, comma or `;` all work. Within one clause, separate fields with commas or newlines: a `;` closes the clause, and the next field is orphaned.\n\n```malloy\n// WRONG: \"no viable alternative at input 'charters'\"\nrun: schools -> { aggregate: total is count(); charters is count() { where: is_charter } }\n// RIGHT\nrun: schools -> { aggregate: total is count(), charters is count() { where: is_charter } }\n```\n\nThe same message appears for an unnamed aggregate after the first: `aggregate: n, max(x)` fails at `max`. Name every entry.\n\n## Compiles, Then Refused by the Warehouse\n\n`Query execution failed: ...` means the Malloy compiled and the warehouse rejected the SQL, often over a type. The position it quotes is in the generated SQL, not in your query. BigQuery, for example, will not partition a window function on a FLOAT64 column. `partition_by:` takes only a field name, so cast in `group_by:` (`season_key is season::string`) and partition on `season_key`, or fix the type in the model."
|
|
323233
323694
|
},
|
|
323234
323695
|
{
|
|
323235
323696
|
name: "malloy-gotchas-rendering",
|
|
@@ -323362,7 +323823,7 @@ view: overview is {
|
|
|
323362
323823
|
{
|
|
323363
323824
|
name: "malloy-html-data-app-runtime",
|
|
323364
323825
|
description: "The JavaScript that drives an in-package HTML data app - calling Publisher.query, building queries from filter state, handling results and errors. For writing the page's data code.",
|
|
323365
|
-
body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a source with `#(authorize)`, a bad given (unknown name or wrong-typed value) fails closed in the authorize check, so it looks like access denied rather than validation. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
|
|
323826
|
+
body: '# HTML Data App Runtime\n\n> `Publisher.query(modelPath, malloy)` returns an array of plain row objects. Build the Malloy string, let the model do the work, and render the rows with whatever front-end code you like.\n\nThe runtime loads from the root-relative `<script src="/sdk/publisher.js">` and adds one global, `window.Publisher`.\n\n## The query contract\n\n| Call | Returns | Use for |\n|---|---|---|\n| `Publisher.query(modelPath, malloy, opts?)` | `Promise<Array>` of rows | driving your own charts and tables |\n| `Publisher.queryFull(modelPath, malloy, opts?)` | `Promise<MalloyResult>` | handing to `<malloy-render>` |\n\n- `modelPath` is the model FILE path within the package, with `/` separators (`"subscriptions.malloy"`, `"models/events.malloy"`). It is not the source name.\n- `malloy` is any query string, written in standard Malloy. This skill covers only the JavaScript glue, not Malloy syntax.\n- `opts` (all optional): `sourceName`, `queryName` (`queryName` runs a saved query, with `sourceName` qualifying the source a view hangs off; `sourceName` on its own is a 400), `givens` (a `{ name: value }` map bound to the model\'s Malloy `given:` runtime parameters for this query; safe parameterization, values are bound by Publisher server-side, not string-interpolated), `filterParams` (values for the model\'s legacy `#(filter)` source filters), `bypassFilters`, and `environment` / `package` (only if the page is served from outside `/environments/<env>/packages/<pkg>/`). `givens` and `filterParams` compose (both apply).\n\n## Structure the app as modules, not one inline script\n\nPast a single tile, an inline `<script>` becomes unmaintainable and untestable. Split the work, and load it without a build step: put your shared libraries first as plain globals, then one ES-module entry point that `import`s your own files.\n\n```html\n<!-- Globals first: the runtime, then any vendored chart library. -->\n<script src="/sdk/publisher.js"></script>\n<script src="./vendor/chart.umd.js"></script>\n<!-- One module entry; it imports the rest. ES modules resolve with no bundler. -->\n<script type="module" src="./app.js"></script>\n```\n\nA separation that keeps each piece testable and changeable on its own:\n\n- **`format.js`**. Pure functions only: number/date formatting, a series-align-by-month helper, status thresholds. No DOM, no globals. This is the file `node --test` can cover directly.\n- **`charts.js`**. Turns a prepared data object into a drawn chart; the only file that touches the chart library.\n- **`tiles.js`**. Your tiles as *data*: for each, its model/source/view (and target source, if any), plus a pure `build(rows)` that shapes query rows for the chart. This is the single source of truth for what each tile queries.\n- **`app.js`**. The thin entry point: reads `tiles.js`, runs the queries, wires results to the DOM. Adding a tile means adding a `tiles.js` entry, not editing `app.js`.\n\nDeclare each tile\'s source and view names once, in `tiles.js`, and have everything else (render code, any agent prompt, tests) read from there. A second copy of those names in another file is the classic drift bug, and a *derived* name (`okr_4_4_2_targets` invented from a tile code) is simply wrong: a target source may have an irregular name or not exist at all. Read the model; don\'t compute names.\n\n## Patterns that work\n\nThese assume a `subscriptions.malloy` model whose `subscriptions` source defines the views `plan_mix`, `mrr_by_industry`, and `kpis`. The names are illustrative; swap in your own model and view names.\n\nRun a named view:\n\n```js\nconst rows = await Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nRefine a view from UI state by appending a `where:`. Restrict the values to ones you control (for example a dropdown populated from the model\'s own distinct values) and escape each interpolated value with a backslash before quotes and backslashes (Malloy rejects the SQL-style `\'\'` doubling). An unescaped apostrophe in a value breaks out of the literal:\n\n```js\nfunction whereClause(state) {\n const q = (s) => s.replace(/\\\\/g, "\\\\\\\\").replace(/\'/g, "\\\\\'"); // backslash-escape for Malloy\n const parts = [];\n if (state.plan) parts.push(`plan = \'${q(state.plan)}\'`);\n if (state.industry) parts.push(`industry = \'${q(state.industry)}\'`);\n return parts.length ? `where: ${parts.join(", ")}` : "";\n}\nconst rows = await Publisher.query(\n "subscriptions.malloy",\n `run: subscriptions -> plan_mix + { ${whereClause(state)} }`,\n);\n```\n\nDo not interpolate free-text or otherwise untrusted input into the query string. Route parameterized input through `opts.givens` (or the legacy `opts.filterParams`) instead: Publisher binds those values server-side as typed parameters, so they are never concatenated into query text and can\'t inject query syntax. (One nuance: a `filter<T>`-typed given takes Malloy filter syntax as its value, so validate it against a known set like any other input; scalar givens carry no syntax at all.) `opts.givens` is safe *parameterization*, not an authorization boundary: a client-supplied given is client-trusted unless a trusted tier upstream sets it from verified identity. Publisher has no per-package control that strips or finalizes one; identity-bound givens are a planned milestone, not a shipped feature. Where you must build query text from input, constrain it to a known set and escape it, or keep the filtering in model-defined views.\n\nKPI or single-row view. Destructure element zero:\n\n```js\nconst [kpis] = await Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis");\nel.textContent = kpis.active_mrr; // the result is an array; kpis.active_mrr, not rows.active_mrr\n```\n\nRefresh a dashboard. Fire the tiles together:\n\n```js\nconst [planMix, byIndustry, kpisRows] = await Promise.all([\n Publisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> mrr_by_industry"),\n Publisher.query("subscriptions.malloy", "run: subscriptions -> kpis"),\n]);\n```\n\nPrefer defining the views in the model (one per tile, pre-aggregated and sorted) over building long query strings in JS.\n\nGet the numbers right. The fastest way to ship a wrong-but-convincing dashboard is to paper over missing data:\n\n- **Missing is not zero.** When you join two series (actuals to a separately-keyed target) and a key is absent, leave it `null` so the chart skips it. Do not `|| 0`, which plots a real-looking zero and reads as "we hit nothing that month." Align on a normalized key (`"YYYY-MM"`), and let the renderer omit null points:\n\n ```js\n // monthKey/monthLabel are your own format.js helpers: monthKey normalizes a\n // date to a "YYYY-MM" string; monthLabel formats it for display.\n // target may not cover every actual month; an absent month stays null, never 0.\n const target = new Map(planRows.map((r) => [monthKey(r.plan_month), Number(r.target_revenue)]));\n const data = actualRows.map((r) => ({\n label: monthLabel(r.order_month),\n actual: Number(r.revenue),\n target: target.has(monthKey(r.order_month)) ? target.get(monthKey(r.order_month)) : null,\n }));\n ```\n\n- **"Current" means latest non-null.** For a KPI scorecard, scan back to the last month that actually has data rather than reading the final row, which may be an incomplete current month.\n- **Guard division in Malloy, not after.** `avg(paid / nullif(active, 0))`. A `nullif` in the query beats catching `Infinity`/`NaN` in JS.\n- **Convert units explicitly.** If the model stores a 0 to 1 fraction and you show a percent, multiply once in `build()` and comment it. Mismatched units are a silent off-by-100.\n\nLoading, empty, and error states. Handle all three; a bare `.then()` that assumes rows leaves the page blank when the query is slow or fails:\n\n```js\nconst el = document.getElementById("out");\nel.textContent = "Loading...";\nPublisher.query("subscriptions.malloy", "run: subscriptions -> plan_mix")\n .then((rows) => {\n if (!rows.length) { el.textContent = "No data."; return; }\n render(rows);\n })\n .catch((err) => {\n el.textContent = `Query failed (${err.status ?? ""}): ${err.response?.message ?? err.message}`;\n });\n```\n\nRender through `<malloy-render>`. `queryFull` returns the full Malloy result envelope (the JSON form of the server\'s result, not a live result object) to hand to the component:\n\n```js\nconst el = document.querySelector("malloy-render");\nel.result = await Publisher.queryFull("subscriptions.malloy", "run: subscriptions -> plan_mix");\n```\n\nPublisher does not serve or bundle `<malloy-render>`; you must obtain a built component bundle matched to your model\'s Malloy version and vendor it into `public/` yourself, then confirm it accepts the envelope as-is. The `storefront` example draws with Chart.js and has no `<malloy-render>` element in its pages, so the component itself is not exercised there. A view tagged in the model (for example `# bar_chart`) drives how it draws.\n\nValidate every query before wiring it into render code, using whatever query tool your environment provides, or by POSTing the query to a running Publisher at `/api/v0/environments/<env>/packages/<pkg>/models/<modelPath>/query` with body `{"compactJson":true,"query":"..."}`, or by running `Publisher.query` once and logging the rows. Malloy names result columns after the `group_by` / `aggregate` field names (`group_by: plan` gives a `plan` column; `aggregate: account_count` gives an `account_count` column), so confirm those names against real output before you read them.\n\nIf you validate the rendered page in a headless browser (Playwright or Puppeteer), do not wait for network idle: `publisher.js` holds the live-reload SSE stream open, so the page never reaches it. Wait on `domcontentloaded` or `load` plus a content selector instead.\n\n## Context, auth, live reload (all automatic)\n\n- Context. A page served under `/environments/<env>/packages/<pkg>/...` infers its environment and package, so `query` needs no env or package args. Serving from elsewhere? Pass `opts.environment` and `opts.package`.\n- Auth. By default the runtime sends cookies (`credentials: include`), so a signed-in user is authenticated with no code. For a bearer token, call `Publisher.setToken(token)` first; `Publisher.setToken(null)` reverts to cookies.\n- Live reload. Under `--watch-env`, the page reloads on package changes by itself. Nothing to wire.\n\n## When the app fails\n\n| Symptom | Likely cause and fix |\n|---|---|\n| 404 or "model not found" | `modelPath` wrong. It is the file path (`"subscriptions.malloy"`), with `/` separators, not the source name. |\n| "source/view not defined" | View or source name guessed. Read the model (your environment\'s context tool, or open the `.malloy` file) and use the real names. |\n| Promise rejects, message starts `Publisher.query:` | Read `error.status` and `error.response` for the server\'s reason (compile error, missing required parameter, permission). |\n| Empty array when you expect rows | Filter value mismatch (case, spelling, type, or a non-ASCII character like `≤` or an en-dash in the literal). Copy the literal verbatim from the model, do not retype the user\'s paraphrase, and confirm it with a distinct-values query (`run: src -> { group_by: the_dimension }`). Quote strings, use `@` for dates. |\n| 400 on a given (ungated source) | An unknown given name (check spelling; names are case-sensitive), a required given left unset, or a value that doesn\'t fit the declared type. Malloy rejects it when preparing the query; supply declared givens via `opts.givens` with the right shape (see the givens type table). |\n| 403 on a query that should be allowed, when passing givens to a gated source | On a gated source, a bad given (unknown name or wrong-typed value) fails closed, so it looks like access denied rather than validation. A `#(authorize)` lock also answers 403 for a caller it simply does not admit. Check the given names and values against the model. |\n| KPI shows `undefined` | The result is an array. Read `rows[0].field` (or destructure `const [k] = ...`), not `rows.field`. |\n| Page loads in dev but is not listed or not served | The file is not under the package\'s `public/` directory. Publisher serves only `public/`; a page written anywhere else (for example `/tmp`) is never reachable at `/environments/<env>/packages/<pkg>/<file>`. |\n| Queries fail only when embedded cross-origin | Cookies are not sent cross-site. Serve same-origin, or pass a bearer token. |\n| No live reload | Watch mode is off. Start with `--watch-env <env>`; without it the events stream reports `mode: disabled` and never reloads. |'
|
|
323366
323827
|
},
|
|
323367
323828
|
{
|
|
323368
323829
|
name: "malloy-html-data-apps",
|
|
@@ -324006,7 +324467,7 @@ LookML flattens nested field names with double underscores: \`project__id\`, \`l
|
|
|
324006
324467
|
{
|
|
324007
324468
|
name: "malloy-lookml-review/curate-visibility",
|
|
324008
324469
|
description: "LookML Visibility → Malloy Access Modifiers (Step 8). Reference detail for the malloy-lookml-review skill.",
|
|
324009
|
-
body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control).
|
|
324470
|
+
body: "# LookML Visibility → Malloy Access Modifiers (Step 8)\n\n> Classify LookML visibility mechanisms and map them to Malloy access modifiers. This runs during Step 8 (CURATE) when the prior-art notes have a Visibility Seeds section.\n\n## LookML Visibility Mechanisms\n\n| LookML Mechanism | Semantics | Strength |\n|---|---|---|\n| `hidden: yes` | Hidden from Explore field picker, but still queryable via URL/API | Cosmetic: UI decluttering |\n| `fields` (explore-level) | Excluded from an explore's available field pool. Not queryable. Can use `ALL_FIELDS*` with `-field` exclusions. | Structural: field genuinely inaccessible |\n| `fields` (join-level) | Restricts which fields from a joined view enter the explore. Include-list only. | Structural: field never enters the pool |\n| `fields_hidden_by_default: yes` | All view fields hidden unless individually opted in with `hidden: no` | Cosmetic: bulk hide |\n| `required_access_grants` | User-attribute-based restriction. True security mechanism. | Security: access controlled |\n\n## Classification → Malloy Treatment\n\nFor each hidden/excluded field, determine the **reason** and map to the correct Malloy treatment:\n\n| Classification | LookML Signal | Malloy Treatment |\n|---|---|---|\n| **Intermediate calculation** | `hidden: yes` + referenced by other fields via `${field}` | `# hidden` tag: keep accessible, hide from display |\n| **Join key / FK** | `hidden: yes` + used in `sql_on:` conditions | Keep public, no `#(doc)`: needed for joins |\n| **UI clutter** | `hidden: yes` + not referenced elsewhere | Assess: if truly unused, `internal:` candidate; if just verbose, `# hidden` |\n| **Explore-scoped exclusion** | Excluded via `fields` parameter | `internal:` candidate: field was structurally inaccessible |\n| **Bulk hidden** | `fields_hidden_by_default: yes` on view | Treat each field individually based on its role |\n| **Access-restricted** | `required_access_grants` | `private:` candidate: flag for user decision |\n| **Not hidden** | No visibility restriction | Public + `#(doc)` candidate |\n\n## Key Distinction\n\nLookML `hidden: yes` ≈ Malloy `# hidden` (cosmetic). LookML `fields` exclusion ≈ Malloy `internal:` (structural).\n\n**Do NOT map `hidden: yes` directly to `internal:`.** That over-restricts fields that may be needed as intermediate calculations or join keys.\n\n## Pattern: Restricted-field masking (`include { private: } + pick`)\n\nWhen a sensitive field should be *present in a redacted/bucketed form* rather than dropped, don't reach for an access modifier alone: modifiers hide or expose, they don't transform. Keep the raw column **`private:`** and derive a masked dimension from it in the source's immediate `extend {}`: a `private:` field is referenceable from the immediate extension, so the raw value stays out of the queryable surface while the mask is public. The mask must key on **data the model can see**, a row-level column or a source `parameter`, not on the viewer:\n\n```malloy\n##! experimental.access_modifiers\nsource: revenue is conn.table('…') include {\n private: raw_amount // raw stays out of the queryable surface…\n} extend {\n dimension: amount is // …but is referenceable here, in the immediate extension\n pick null when is_confidential // is_confidential is a ROW column, not the viewer\n else raw_amount\n // or bucket instead of null-out:\n // pick 'under 50k' when raw_amount < 50000\n // pick '50k–100k' when raw_amount < 100000\n // else '100k+'\n}\n```\n\nGive the mask a **different name** from the private raw column (`amount` vs `raw_amount`), reusing the name is a redefinition. If the mask must carry the raw column's *exact* original name, that forces a `rename:`, which pushes you onto the `extend { rename }` path. That path composes with `include {}`, but only in one order (the `extend { rename: }` first, then `include {}` naming the post-rename field; see `skill:malloy-gotchas-modeling`), and you still have to move the raw column out of the way before the mask can take its name. Prefer a distinct mask name and keep the raw `private:` unless the exact original name is required.\n\n**This is NOT per-viewer access control.** LookML `required_access_grants` gates on the *viewing user's* attributes; Malloy models have no per-viewer context, so a model-layer `pick` can only redact based on row data or a parameter. To gate a field by *caller identity/role* (the real `required_access_grants` equivalent), use `#(authorize)` on the source, driven by trusted attributes in Malloy Publisher (see `skill:malloy-model` § Access Control). `#(authorize)` refuses a caller it does not admit with a 403; `#(access_filter)` is the row-scoping sibling. Masking and these gates are independent layers; use the mask for \"everyone sees a coarsened value,\" a gate for \"only some callers see the field at all.\"\n\n## Pattern: Long→wide custom-field pivot\n\nSee `build-derived-tables.md` → \"Long→wide entity-values pivot\" for turning an entity-attribute-value (EAV) custom-field table into wide columns with **filtered aggregates**, the right move when LookML modeled N per-attribute joins.\n\n## Process\n\n1. Read the Visibility Seeds notes captured during discovery\n2. For each field, apply the classification rules above\n3. Present the classified list to the user for confirmation before applying access modifiers"
|
|
324010
324471
|
},
|
|
324011
324472
|
{
|
|
324012
324473
|
name: "malloy-lookml-review/discover",
|
|
@@ -324234,12 +324695,12 @@ Present coverage in this order:
|
|
|
324234
324695
|
{
|
|
324235
324696
|
name: "malloy-materialization",
|
|
324236
324697
|
description: "Add and debug Malloy Persistence materializations in a package - persist an expensive source so queries read a pre-built table. Read this whenever the user wants to materialize a source, add a persist annotation, speed up a slow source, or asks why a persist source isn't building.",
|
|
324237
|
-
body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting an `#(authorize)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **An `#(authorize)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting an `#(authorize)`-gated source**."
|
|
324698
|
+
body: "# Materialization (Malloy Persistence)\n\nMaterialize an expensive source once so queries read a **pre-built warehouse table** instead of recomputing it every time. You tag a source `#@ persist`, a materialization run builds it into a physical table, and queries against it are rewritten to read that table.\n\n> **The #1 gotcha, up front:** if a persist source isn't materializing, it is almost always one of two things - a `.malloy` file in the package missing the `##! experimental.persistence` flag (which aborts the *whole* package's build plan), or no build ever ran (a standalone Publisher does not build on publish - see **Building and refreshing**). Jump to **Debugging a no-op build**.\n\n## The recipe (get this right and it just works)\n\n1. **`##! experimental.persistence` on EVERY `.malloy` file in the package** - not only the file that declares the persist source. Either form enables it:\n - `##! experimental.persistence`, or\n - `##! experimental { access_modifiers, sql_functions, persistence }` (add `persistence` to the existing list).\n\n **Why every file:** the build plan is computed by asking *every* `.malloy` file in the package for its persist sources, and that call **throws on any file whose model lacks the flag** (`Model must have ##! experimental.persistence`). One unflagged helper or import file, even one with no persist source of its own, aborts the whole package's build plan, so *every* persist source in the package drops out. This is the most common cause of a no-op build.\n\n2. **`#@ persist name=\"...\"` on a query-based source, with the name quoted:**\n ```malloy\n #@ persist name=\"my_dataset.my_table\"\n source: my_rollup is some_source -> { group_by: ...; aggregate: ... }\n ```\n - **Only `query_source` and `sql_select` sources are persistable** - a source whose definition has a `-> { ... }` pipeline or a `conn.sql(\"...\")`. This **includes** one refined by a trailing `extend { ... }`. What is **not** persistable is a *plain* `extend` over a bare `conn.table(...)`; a `#@ persist` on such a source is **silently ignored** (its annotation is never read) - that one source just won't materialize, and the rest of the package still builds.\n - **Quote the name.** `name=\"my_table\"` (or a path `name=\"dataset.table\"` / `name=\"project.dataset.table\"`) is required. A **bare** `name=my_table` **always fails the build/publish** with `persist annotation name must be quoted` (a raw-source scan that hard-stops); it never silently no-ops.\n - `name=` is the target table name. In a standalone Publisher this **is** the physical table (rebuilt in place); a hosted (control-plane) deployment builds it under a content-addressed generation name. In both, the source's identity for reuse is a content address of its connection and canonical SQL (its `sourceEntityId`), so **republishing unchanged persist logic reuses the existing table** and changing the logic builds fresh.\n\n3. **Package persistence policy in `publisher.json`** (all optional):\n ```jsonc\n {\n \"name\": \"my-package\",\n \"materialization\": {\n \"scope\": \"package\", // default; \"version\" = each published version owns its own tables\n \"freshness\": { \"window\": \"24h\", \"fallback\": \"live\" },\n \"queryMetadata\": { \"team\": \"finance\" } // tags the build's backend statements\n }\n }\n ```\n Enforced at publish (strict), on edits (strict), at load (warn, still serves), and by the scheduler (an offending package is skipped):\n - **`scope`**: `package` (default; artifacts reused across published versions) or `version` (each artifact owned by one version). Package-level only; there is no per-source scope. A root-level `scope` is the deprecated home and still works, with a warning; declaring both homes with different values is rejected.\n - **`materialization.freshness`** (`window` + `fallback` of `live`/`stale_ok`/`fail`) is the objective a **hosted control plane** enforces by refreshing the table to meet it (`fallback: \"live\"` serves live compute while stale/absent). A **standalone** Publisher does **not** act on `freshness` for refresh - see **Building and refreshing**.\n - **`materialization.queryMetadata`** is a bag of string properties attached to every statement the build issues, for the backend's own cost attribution (Snowflake `QUERY_TAG`, BigQuery job labels, a leading SQL comment elsewhere). Overridable per source with `#@ persist queryMetadata.<name>=\"<value>\"`. Observability only: it never changes what gets built. See `docs/query-metadata.md`.\n - **`materialization.schedule`** is a 5-field UTC cron (`min hour dom mon dow`; `L`/`W`/`#`/`?` rejected). It **requires `scope: \"version\"`** and is **mutually exclusive with `freshness`**. This is how a standalone Publisher refreshes on a cadence.\n\n4. **Reads vs writes.** The persist source can *read* any dataset the connection can read; the persist *target* (`name=`'s dataset) must be a dataset the connection can **write** (typically a scratch dataset).\n\n## Building and refreshing (standalone vs. hosted)\n\nA `#@ persist` tag declares *what* to materialize; it does not by itself build anything.\n\n- **Standalone Publisher:** publishing or loading a package only computes its build plan - **no table is built until a materialization run executes.** Trigger one explicitly (`malloy-pub materialize --package <pkg> --wait`, or the materialization API), or turn on the opt-in local scheduler (off unless `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is set) to fire the package's `schedule` cron. Refresh is a re-run or that cron; `freshness` is not a refresh trigger here, so a freshness-only standalone package builds once and is not auto-refreshed.\n- **Hosted (control-plane) deployment:** the build runs automatically on publish, best-effort - a build failure does **not** fail the publish (which is why a broken persist can look like a silent no-op), and the control plane drives refresh to meet the `freshness` objective.\n\nEither way, a successful publish alone does not prove a table exists - confirm the build separately.\n\n## Serve-time routing is `query_source`-only (today)\n\nBoth persistable types *build* a table, but only a **`query_source`** (a `-> { ... }` pipeline) is rewritten to *read* it at query time. A raw **`sql_select`** (`conn.sql(\"...\")`, including `conn.sql(\"...\") extend { ... }`) builds its table and then the query path re-inlines its SQL, so the table is built and never read, and queries are no faster. If you have raw SQL you want served from a table, wrap it in a thin `query_source` and persist that:\n\n```malloy\nsource: x_raw is my_conn.sql(\"select ...\")\n#@ persist name=\"scratch_dataset.x\"\nsource: x is x_raw -> { select: * }\n```\n\n## Confirming it worked\n\nAfter a build runs, re-run one of the source's queries - a persisted `query_source` should return quickly, reading the pre-built table instead of recomputing the upstream. Your host also reports each persisted source as **ready** with its physical table name (a materialization run detail, CLI listing, or materialization view, depending on the host); if nothing is listed, either no build ran (standalone) or the build plan was empty - see **Debugging a no-op build**.\n\n## Debugging a no-op build\n\nSymptom: no table was built and the source still recomputes on every query. Check, in order:\n\n0. **Did a build actually run?** On a standalone Publisher, publish/load does **not** build - run `malloy-pub materialize` (or enable the scheduler). \"Publishes fine, no table\" is the *expected* standalone state, not a model bug. On a hosted deployment the build is automatic but best-effort, so a failure is silent - look for a `FAILED` run.\n1. **A `.malloy` file missing the persistence flag** (the most common real bug). Every model file's `##!` line needs `persistence`, including pure helper/import files with no persist source - one unflagged file aborts the whole package's build plan.\n2. **An unquoted persist name** - a bare `name=foo` **always** hard-stops the build/publish with `persist annotation name must be quoted`; use `name=\"foo\"`. (If you got *no* error at all, it isn't this.)\n3. **A `#@ persist` on a non-persistable source** - a bare `extend` over `conn.table(...)` is silently ignored, so *that* source won't materialize (the rest of the package is unaffected). Tag a `query_source` / `sql_select` instead.\n4. **A persisted raw `sql_select` that builds but is never read** - if the table exists yet queries are no faster, it's the serve-routing gap above; wrap the `sql_select` in a `query_source`.\n\n**Isolation test** - add a trivial, self-contained persist source in its own file and rebuild:\n```malloy\n##! experimental.persistence\nsource: smoke_raw is my_conn.table('some_dataset.some_table')\n#@ persist name=\"scratch_dataset.persist_smoke_test\"\nsource: persist_smoke is smoke_raw -> { aggregate: n is count() }\n```\n- If **even this** doesn't build (after a real materialization run), the whole package's plan is aborting - a sibling `.malloy` file is missing the flag. Fix rule 1 across the package.\n- If the smoke source **does** build but your real one doesn't, your real source is the problem - a non-persistable type (a bare `extend`), or its own file's flag.\n\nDelete the smoke file and drop its table afterward.\n\n## Persisting a `#(access_filter)`-gated source\n\nA gated source **can** be persisted, but only on one tier and only in one shape, and the thing to be\ncareful about is not refused by anything - you have to decide it.\n\n- **`storage=` and `#@ preaggregate` always refuse a gated source**, with a 422 at build time naming the\n source. (`#@ persist storage=<name>` is the tier that materializes into a separate registered storage\n destination and serves from there, rather than building in the source's own connection; `#@ preaggregate`\n stores a rollup Publisher derives from a measure you annotated with a grain, rather than a source you\n wrote.) A rollup also groups *across* the gated column, so it could not be row-filtered afterwards even\n in principle.\n- **A colocated `#@ persist` (no `storage=`) is admitted** when the gate is provably the entry point's\n **own row filter**. It is refused when the gate is reached only through a join, inherited from a base\n the compiler cannot attribute cleanly, or does not classify as a row filter at all. The gate is found\n through the import -> rename -> `query_source` chain, so a gate the persisted source did not declare\n itself still counts.\n\n**What to be wary of.** Persisting does not weaken the gate: it changes only where rows are read FROM, and\nthe gate still runs live on every query as that query's own `WHERE`, so filtered rows come back filtered.\nWhat freezes is the **column the gate filters on**. A row whose access decision changes - it changes\nowner, say - keeps being served under its OLD decision until the next rebuild. That is a stale *access\ndecision*, not merely stale data, and nothing raises an error.\n\n**None of this is needed for the gate to work.** It is enforced live on every query either way; what\nneeds a bound is how long a *stale* decision can survive. Of the three controls that look like that\nbound, only the first is:\n\n- **`materialization.freshness` `{ \"window\": \"24h\", \"fallback\": \"live\" }` is the bound.** The serve path\n re-checks freshness per query, so once the artifact ages past the window it drops out of the serving set\n and the query recomputes live, correctly filtered - whether or not a rebuild ever lands. Three details\n decide whether you actually get that. **`fallback` must be `live`**: under `stale_ok` a stale artifact\n keeps being served, which voids the bound, and window and fallback resolve *independently* per layer,\n so a package-level `stale_ok` silently defeats a window you set on the source. That is a statement about\n **layers**, which do not combine - not about siblings, below. Prefer the\n **per-source** spelling `#@ persist name=\"...\" freshness.window=\"24h\" freshness.fallback=\"live\"` over\n the package-wide `materialization.freshness` key: the gated source is what needs the bound, and setting\n it package-wide forces every other persisted source to recompute once stale too. And **a\n content-identical sibling shares the artifact, so it shares the window**: reuse is keyed on the\n content-addressed `sourceEntityId`, which folds the connection and the SQL but *not* the source name, so\n two persist sources whose bodies compute the same SQL resolve to one table carrying one freshness\n policy. The tightest window any of them declares governs all of them - a sibling declaring nothing\n cannot loosen yours, and yours pulls that sibling's reads off the table once it lapses. A sibling's\n `stale_ok` cannot void your bound either: the fold keeps whichever fallback bounds staleness, so the\n layer rule above does not carry over here. If two sources need genuinely different windows, give them\n genuinely different SQL.\n\n Both of those are properties of the **host** that assembles the manifest, not of the annotation. Where\n the host does not fold, which sibling's policy reaches the wire is unspecified; and a host that folds at\n manifest-assembly time typically applies it when a version's manifest is next published rather than\n retroactively to manifests already distributed - so you can declare the window correctly and not have it\n in force yet.\n- **A cron alone is not a bound.** A failed build or a stopped scheduler leaves the source serving its old\n decisions indefinitely. `freshness` and `schedule` are mutually exclusive; for a gated source, take the\n window.\n- **`refresh=\"incremental\"` does not bound revocation.** The delta only re-reads rows in\n `[covered_through, frontier)`, so a row that changes owner *without its watermark advancing* is never\n re-read - while the entry still reports an advancing `coveredThrough` and reads as healthy. Only a full\n rebuild recomputes the gating column.\n\n**And the window only binds where the serving manifest carries it.** Freshness is enforced from fields a\ncontrol plane stamps onto the manifest it distributes; a Publisher that serves what it just built binds the\ntable with no `dataAsOf` and no window, and an entry carrying no window never ages out. So on a standalone\ndeployment the declared window is inert and the artifact serves until the next full rebuild - which leaves a\nrebuild cadence you actually verify as the only bound, and makes leaving a revocation-sensitive source\nunpersisted the safer call.\n\nWhen recommending `#@ persist` on a gated source, pair it with a freshness window and say out loud what\nstaleness the author is accepting. A gated source with neither a window nor a full-rebuild cadence has no\nbound on how long a revoked row keeps being served.\n\n## Gotchas\n\n- **Every `.malloy` file needs the persistence flag** - one unflagged file aborts the whole package's build plan. (A `#@ persist` on a *non*-persistable source, by contrast, is silently ignored and does not affect other sources.)\n- **A tag doesn't build** - a standalone Publisher materializes only on an explicit run or its scheduler; only a hosted control plane builds on publish.\n- **Serve-time routing is `query_source`-only** - a raw `sql_select` builds a table the query path doesn't read; wrap it in a `query_source`.\n- **Quote the name** - a bare `name=` always hard-stops the build.\n- **Republishing unchanged persist logic reuses the table** - reuse is keyed on the content-addressed `sourceEntityId`, not the `name=`.\n- **Removing a persist source (or a smoke test) does not drop its table** - physical-table cleanup is the caller's responsibility; drop it yourself.\n- **A `#(access_filter)`-gated source freezes its gating column when persisted** - the gate still runs live, but a revoked row keeps being served under its old access decision until the next rebuild. `storage=` and `#@ preaggregate` refuse a gated source outright. See **Persisting a `#(access_filter)`-gated source**."
|
|
324238
324699
|
},
|
|
324239
324700
|
{
|
|
324240
324701
|
name: "malloy-materialization-tuning",
|
|
324241
324702
|
description: "Optimize a package's Malloy Persistence materializations for cost and performance using the malloy-pub CLI and the materialization history. Recommend what to persist, what to stop persisting, and how to schedule/scope it. Use when the user asks to make a package cheaper or faster, tune persistence, decide what to materialize, or review persist/schedule choices.",
|
|
324242
|
-
body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries
|
|
324703
|
+
body: "# Tuning materializations for cost and performance\n\nThis skill turns the signals the open-source Publisher already records (the materialization history, per-run timings, and which sources were built vs reused) into concrete, **recommendations-only** advice: which sources to persist, which to stop persisting, and how to schedule and scope them. It is the local counterpart to the platform's usage-driven optimization: the Publisher has the raw signals, and you read them with the `malloy-pub` CLI.\n\n> **Recommendations only. Never change a model, schedule, or scope without the user's explicit go-ahead.** Present the findings and the proposed edits, then apply them only when asked. Persisting the wrong source wastes storage and rebuild time; unpersisting a hot one makes queries slow. Let the user decide.\n\nAssumes the `malloy-pub` CLI is on PATH and points at the server (`--url` or `MALLOY_PUBLISHER_URL`, default `http://localhost:4000`). Substitute the real environment and package for `<env>` / `<pkg>`.\n\n## Step 1: Take inventory\n\nEstablish what the package persists today and how it is governed.\n\n- **Persist sources:** the sources annotated `#@ persist name=\"…\"` in the package's `.malloy` files. Read the models (or `get_context` the package) to list them.\n- **Schedule + scope:**\n\n ```bash\n malloy-pub schedule view --environment <env> --package <pkg>\n ```\n\n This prints the cron (or `none`, meaning publish / on-demand only), the persist **scope** (`package` = artifacts reused across versions; `version` = per published version), and whether a freshness policy is set. A control-plane-managed package (manifestLocation set) is refreshed by the control plane, not the standalone scheduler, so leave its cadence alone.\n\n## Step 2: Read the materialization history\n\nThe history is where cost lives. Each run records its trigger, timing, and how many sources were built vs reused.\n\n- **For one package** (newest first). A materialization is a run of one package's persist sources, so this is the listing; to review a whole environment, read each package in turn (`malloy-pub list package --environment <env>` names them):\n\n ```bash\n malloy-pub list materialization --environment <env> --package <pkg>\n ```\n\n Columns: ID, Status, **Trigger** (`SCHEDULER` vs `ON_DEMAND`), Started, Completed, Error.\n\n- **A single run's detail** (the cost signals):\n\n ```bash\n malloy-pub get materialization <id> --environment <env> --package <pkg>\n ```\n\n In the JSON, read:\n - `metadata.durationMs`: how long the build took.\n - `metadata.sourcesBuilt` vs `metadata.sourcesReused`: how much work each run actually did. A run that is nearly all _reused_ is cheap; one that is nearly all _built_ every time is where cost accumulates.\n - `metadata.trigger`: `SCHEDULER` (a cron fired it) or `ON_DEMAND`.\n - `manifest.entries[*]`: the persisted sources: `sourceName`, `physicalTableName`, `realization`.\n\nLook across several runs, not one: the pattern over the recent history (how often it rebuilds, how much it reuses, how long it takes) is the signal.\n\n## Step 3: Analyze and recommend\n\nWeigh rebuild cost against query benefit. Common findings:\n\n- **Persist candidate:** an expensive, frequently-queried source that is _not_ persisted (recomputed on every query). Recommend adding `#@ persist name=\"…\"`. Strongest when the source is a heavy aggregate/join reused by many queries and its inputs change slowly. If the source carries a `#(access_filter)` gate, check which case it is first. A gate reached only through a `join_*`, or inherited from a base it cannot be attributed to, makes the colocated persist **refuse with a 422** - so do not recommend persisting it at all. Where the gate is provably the source's own row filter the persist is admitted, and then only recommend it alongside a freshness window (`freshness.fallback=\"live\"`): the gating column freezes at build time, so without one a revoked row can be served under its old access decision indefinitely (see `skill:malloy-materialization`, which also covers where that window does and does not bind).\n- **Removal candidate:** a persisted source that is cheap to compute, rarely queried, or rebuilt far more often than it is read. Recommend dropping the `#@ persist` annotation (and its table): the storage + rebuild cost is not buying anything.\n- **Cadence mismatch:** a `SCHEDULER` cadence out of step with how fast the data changes or how long a build takes. If most scheduled runs are all-reused (nothing changed), the cron is too frequent, so loosen it. If queries routinely read stale data, tighten it. If the build's `durationMs` approaches the interval, the cadence is too aggressive.\n- **Scope mismatch:** `scope: version` re-materializes per published version (right when versions must be isolated, e.g. a schedule); `scope: package` reuses one lineage across versions (cheaper when versions can share). A schedule _requires_ `version`. If a package carries a schedule it does not need, clearing the schedule frees it to use the cheaper package scope.\n\nFrame each recommendation with the evidence from Step 2 (the run IDs, timings, built/reused counts) so the user can judge it.\n\n## Step 4: Apply (only once approved)\n\n- **Add a persist source:** add the `#@ persist` annotation in the `.malloy` file (use the modeling workflow to validate and reload), then rebuild so it is materialized:\n\n ```bash\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n- **Remove a persist source:** delete the `#@ persist` annotation, then drop the old run's tables **before** rebuilding what remains:\n\n ```bash\n malloy-pub delete materialization <id> --environment <env> --package <pkg> --drop-tables\n malloy-pub materialize --environment <env> --package <pkg> --wait\n ```\n\n > `--drop-tables` drops **every** physical table in that run's manifest, not just the removed source's: auto-run assigns stable table names and carries unchanged sources forward, so an old run's manifest names tables a newer manifest still serves. Do the drop **first, then rebuild**: `materialize --wait` re-creates every source that is still persisted, ending in the desired state. Dropping a run whose tables the current serving manifest depends on, without an immediate rebuild, breaks queries (`Table ... does not exist`).\n\n- **Change the schedule / cadence:**\n\n ```bash\n malloy-pub schedule set \"0 6 * * *\" --environment <env> --package <pkg> # 5-field UTC cron\n malloy-pub schedule clear --environment <env> --package <pkg>\n ```\n\n `set` also sets `scope: version` (a schedule requires it); the server rejects an invalid cron or an illegal scope/freshness combination, so a rejection means the change was unsafe.\n\n- **Change the scope:** `scope` is declared in the package's `publisher.json` under `materialization` (`\"scope\": \"package\"` or `\"version\"`; the root-level form is deprecated but still read). Edit it there, then reload/republish the package. Remember a schedule pins scope to `version`, so clear the schedule first if moving to `package`.\n\nAfter applying, re-run Step 2 on the next few builds to confirm the change did what you predicted (more reuse, shorter builds, or a table that is actually read).\n\n## What this skill does not do\n\n- It does not decide for the user or apply changes silently; every edit needs an explicit go-ahead.\n- It cannot see per-query read counts (the open-source Publisher records build history, not query-level table usage), so \"rarely queried\" is a judgment from the model and the user's knowledge, not a measured metric. Say so when it matters."
|
|
324243
324704
|
},
|
|
324244
324705
|
{
|
|
324245
324706
|
name: "malloy-model",
|
|
@@ -324461,7 +324922,7 @@ A given is **declared bare** but **referenced with a \`$\` sigil** in expression
|
|
|
324461
324922
|
- **Givens don't auto-inject a \`where:\`.** Unlike \`#(filter)\`, you write the filter expression that references the given yourself (e.g. \`where: dimension ~ $given_name\`).
|
|
324462
324923
|
- **Not every filter maps cleanly.** A filter with no neutral match-all literal default - e.g. a scalar date/number range like \`> @2020-01-01\` - is not a good \`given:\`; keep those on \`#(filter)\`. Two more cases keep using \`#(filter)\`: mandatory scoping filters (\`required\`) and system-injected row-level filters (\`implicit\`), both below.
|
|
324463
324924
|
|
|
324464
|
-
Givens are also the substrate for access control - see "Access Control:
|
|
324925
|
+
Givens are also the substrate for access control - see "Access Control: \`#(authorize)\` and \`#(access_filter)\`" below.
|
|
324465
324926
|
|
|
324466
324927
|
## Legacy: Parameterizable Filters with \`#(filter)\`
|
|
324467
324928
|
|
|
@@ -324532,38 +324993,72 @@ Publisher formats values based on the dimension's data type, \`string\` → \`'v
|
|
|
324532
324993
|
|
|
324533
324994
|
Pass \`bypass_filters=true\` (REST) or \`bypassFilters: true\` (POST body) to skip filter injection entirely. Use sparingly, required-filter governance only works if bypass is restricted to trusted callers.
|
|
324534
324995
|
|
|
324535
|
-
## Access Control:
|
|
324996
|
+
## Access Control: \`#(authorize)\` and \`#(access_filter)\`
|
|
324997
|
+
|
|
324998
|
+
Gate query access to a source over declared \`given:\` values (\`given:\` is Malloy's native runtime-parameter mechanism, the going-forward replacement for \`#(filter)\`). **Two annotations, one question each, and the name tells you which:**
|
|
324536
324999
|
|
|
324537
|
-
|
|
325000
|
+
| Annotation | The question | Body it takes | A denial is |
|
|
325001
|
+
| --- | --- | --- | --- |
|
|
325002
|
+
| \`#(authorize)\` | may this caller reach this source at all? | \`'literal' <op> $GIVEN\`, plus the \`true\`/\`false\` sentinels | **403** |
|
|
325003
|
+
| \`#(access_filter)\` | which rows may they see, once they may? | \`field_path <op> $GIVEN\` | **200**, with their rows |
|
|
325004
|
+
|
|
325005
|
+
Either is an annotation on its own line directly above the \`source:\` line, carrying a **narrow grammar publisher parses itself, not an arbitrary Malloy expression**: one or more terms joined only by \`and\`, with \`<op>\` fixed by the given's declared arity (\`in\` for a list, \`=\` for a scalar).
|
|
325006
|
+
|
|
325007
|
+
**Nothing is inferred from the body.** The annotation you write declares which question you are answering, and a body that does not fit is refused at load naming the other annotation. A source with no annotation of either kind, own or inherited, is unrestricted.
|
|
325008
|
+
|
|
325009
|
+
The lock is DECIDED, before the caller's query compiles: a caller it does not admit gets a 403, never a row count or a \`NULL\` aggregate over data they were refused. The filter is GRAFTED onto the query as a \`where:\`, so a caller it matches nowhere gets a 200 with an empty result. The lock runs first; a caller it refuses never reaches the filter. A 403 also covers either gate failing to apply at all (a field the entry point dropped, a given nobody supplied).
|
|
324538
325010
|
|
|
324539
325011
|
\`\`\`malloy
|
|
324540
325012
|
##! experimental.givens
|
|
324541
325013
|
|
|
324542
325014
|
given:
|
|
324543
325015
|
ROLE :: string
|
|
325016
|
+
ORG_IDS :: string[]
|
|
324544
325017
|
|
|
324545
|
-
#(authorize)
|
|
325018
|
+
#(authorize) 'analyst' = $ROLE
|
|
325019
|
+
#(access_filter) org_id in $ORG_IDS
|
|
324546
325020
|
source: orders is duckdb.table('orders.parquet') extend {
|
|
324547
325021
|
measure: order_count is count()
|
|
324548
325022
|
}
|
|
324549
325023
|
\`\`\`
|
|
324550
325024
|
|
|
324551
|
-
- **
|
|
324552
|
-
- **
|
|
324553
|
-
-
|
|
324554
|
-
- **
|
|
324555
|
-
- **
|
|
324556
|
-
- **
|
|
324557
|
-
|
|
324558
|
-
|
|
324559
|
-
|
|
325025
|
+
- **Nothing outside the two term shapes above parses.** No \`or\`, \`not\`, \`!=\`, \`<\`/\`>\`/\`<=\`/\`>=\`, function calls, bare field/boolean references, or a literal on the right of a row-level term. \`org_id in $GROUPS\` and \`region = $REGION and org_id in $GROUPS\` are legal; \`upper(region) = $REGION\`, \`(org_id in $GROUPS or region = $REGION)\`, and a bare \`#(access_filter) authorized\` are all refused at load with a named cause (see your deployment's reference documentation for the full list).
|
|
325026
|
+
- **Writing a term on the wrong annotation is refused, both ways.** \`#(authorize) org_id in $GROUPS\` is refused naming \`#(access_filter)\`; \`#(access_filter) 'finance' in $GROUPS\` is refused naming \`#(authorize)\`. The second matters: a constant predicate grafted as a row filter would serve a refused caller 200 with zero rows, which is the answer the lock exists to replace.
|
|
325027
|
+
- **A source may declare more than one note on a route: repeats AND together.** \`#(access_filter) region = $REGION\` stacked with a second \`#(access_filter) org_id in $GROUPS\` both apply, and a caller must satisfy every term across every note. \`or\` is still refused wherever it appears, so there is no way to spell "admit if either" inside one gate or across a source's gates; use two extension sources instead, one per admitted population (see the admin pattern below).
|
|
325028
|
+
- **Two exceptions to "every term references a given", and both live on the lock.** \`#(authorize) false\` is a deny-all: that is how you lock a base. \`#(authorize) true\` is an admit-all, and it is what an extension of a locked base needs in order to be open: a source declaring no gate of its own **inherits** its ancestor's, so leaving the annotation off an extension of a \`false\` base inherits the lock rather than lifting it. Neither sentinel is legal on \`#(access_filter)\`. \`false\` may not share a source with another note at all; \`true\` may not share the lock route with one, but is live beside an \`#(access_filter)\` (\`#(authorize) true\` with \`#(access_filter) org_id in $GROUPS\` re-opens a locked base while still scoping the rows). Neither is a term: \`true and org_id in $GROUPS\` is read as an ordinary two-term body whose first term is malformed.
|
|
325029
|
+
- **Inheritance is per route.** Both inherit through \`extend\` the same way, own wins over ancestor on that route only, so a source can be "own" for one and "inherited" for the other. The API reports them separately, each under the field named for its own annotation: \`Source.authorize\` and \`Source.accessFilter\`.
|
|
325030
|
+
- **Generate the admin escape hatch as another extension over the locked base, never as a bypass.** When a model needs an ordinary population plus a wider one, add a second extension whose rule is a lock on the caller:
|
|
325031
|
+
|
|
325032
|
+
\`\`\`malloy
|
|
325033
|
+
given:
|
|
325034
|
+
ORG_IDS :: string[]
|
|
325035
|
+
GROUPS :: string[]
|
|
325036
|
+
|
|
325037
|
+
#(authorize) false
|
|
325038
|
+
source: orders_base is duckdb.table('orders.parquet') extend {}
|
|
325039
|
+
|
|
325040
|
+
#(authorize) true
|
|
325041
|
+
#(access_filter) org_id in $ORG_IDS
|
|
325042
|
+
source: orders is orders_base extend {}
|
|
325043
|
+
|
|
325044
|
+
#(authorize) 'admin' in $GROUPS
|
|
325045
|
+
source: orders_admin is orders_base extend {}
|
|
325046
|
+
\`\`\`
|
|
325047
|
+
|
|
325048
|
+
Each extension replaces the base's \`false\` on the lock route with its own rule. \`orders\` is open to everyone and scoped per row; \`orders_admin\` is locked to the admin group and, carrying no filter, serves every row to whoever passes. This is the shape to generate whenever an author wants a role to see everything: another extension source, not a flag that skips the gate.
|
|
325049
|
+
- **Both only gate from the \`source:\` line.** The same annotation on a \`dimension:\`/\`measure:\`/\`join_*:\`/\`view:\` line, or on a top-level \`query:\`, is refused at load naming the position rather than silently protecting nothing.
|
|
325050
|
+
- **Every given the gate references must be declared on the entry model's own surface, and must carry no default.** A given the model cannot resolve is refused at load. So is a referenced given declared *with* a default: a caller who supplies nothing would get that default and be admitted or excluded by a value the gate's own line never shows, so it is refused rather than reasoned about case by case.
|
|
325051
|
+
- **A scalar/array mismatch between the operator and the given's declared type is a load-time refusal**, not a request-time warehouse error: \`org_id in $GROUPS\` requires \`GROUPS\` to be array-typed, \`region = $REGION\` requires \`REGION\` scalar. Negation is likewise refused outright, so there is no empty-given inversion surprise to warn about.
|
|
325052
|
+
- **Entry point only: not joined, but inherited through \`extend\`.** The gate applies to the source a query enters through. A gate on a source reached only via \`join_*\` **never fires**, at any depth, so anything ungated that joins a locked base hands the base's rows to every caller. A source that \`extend\`s a locked base and declares no gate of its own **does** carry the base's gate; declaring its own annotation replaces it. A source derived from a locked base via a query (\`source: z is locked -> { … }\`) instead **always carries the base's gate in addition to its own**: the derivation recurses into the base unconditionally, so an own gate does not replace it, and the two combine as separate AND'd entries. Pair a locked base with curated extension sources, using access modifiers (\`include { public: …, private: * }\`), so an extension re-exposes only a curated column surface, and keep sensitive sources out of ungated joins.
|
|
325053
|
+
- **A derivation that drops a column the gate reads fails CLOSED.** \`extend { except: org_id }\`, or an \`accept:\` that omits it, leaves the grafted filter unable to compile, so the request is denied rather than served ungated. The one hole to know: dropping the gated column and then \`rename:\`-ing a *different* column onto that exact name grafts successfully and binds the gate to the wrong column. Narrow, but real, so don't recycle a gated column's name. If a derivation's own projection needs to drop the column a row-level term would read, use a source-level term (\`'literal' in/= $GIVEN\`) instead, since it never depends on any projected column.
|
|
325054
|
+
- **The quoted-string and file-level forms are refused at load and no longer exist.** A quoted expression on the \`source:\` line, in either quote, a file-level \`##(…) "<expr>"\` applying to every source in the file, and the earlier \`internal dimension: authorized is <expr>\` form are all retired; the load names the rewrite. Every \`.malloy\` file in a package compiles at load and any failure aborts the package, so a retired-form gate anywhere in the package is refused. Only a declaring file *outside* the package escapes that: it loads and denies every request instead, with no compile-time hint. See your deployment's reference documentation.
|
|
324560
325055
|
- **A gated source can be persisted, but the gating column freezes.** \`storage=\` and \`#@ preaggregate\` refuse a gated source outright; a colocated \`#@ persist\` is admitted when the gate is provably the entry point's own row filter. The gate still runs live on every query, so rows come back filtered - but the column it filters ON is frozen at build time, so a row whose access decision changes keeps being served under the old decision until the next rebuild. Pair \`#@ persist\` on a gated source with a freshness window (\`fallback="live"\`), which is the only control that bounds that - and read \`skill:malloy-materialization\` for where that window binds, because on a standalone Publisher it does not.
|
|
324561
325056
|
|
|
324562
|
-
> **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`.
|
|
325057
|
+
> **Trust caveat.** Givens are **caller-asserted**, anyone who can reach the query API can claim a favorable given, e.g. \`{"ROLE":"admin"}\`. Neither annotation is a real boundary unless it sits behind a trusted tier that sets givens from its own verified context, never directly from an untrusted caller. Neither is, on its own, end-user authentication.
|
|
324563
325058
|
>
|
|
324564
|
-
> **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning
|
|
325059
|
+
> **Forward direction.** Givens are how access control is built here, and the planned next step is **identity-bound ("secure") givens** - reserved values a trusted tier populates from a verified token or proxy header, which the caller cannot override - turning these gates into a standalone boundary. Model access on \`given:\` + these two annotations now; it is the surface that carries forward.
|
|
324565
325060
|
|
|
324566
|
-
Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's
|
|
325061
|
+
Full syntax, inheritance rules, validation, and the error contract are covered in your deployment's authorize reference documentation.
|
|
324567
325062
|
|
|
324568
325063
|
## Join Syntax
|
|
324569
325064
|
|
|
@@ -325216,7 +325711,7 @@ Steps to follow when the user asks a question:
|
|
|
325216
325711
|
{
|
|
325217
325712
|
name: "malloy-notebooks",
|
|
325218
325713
|
description: 'Create Malloy notebooks (.malloynb) for interactive dashboards and data stories. Use when user asks to "create a notebook", "make a dashboard notebook", "write a malloynb", "data story", or needs to build reports/visualizations.',
|
|
325219
|
-
body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
|
|
325714
|
+
body: "# Malloy Notebooks\n\nFiles: `.malloynb`\n\n## Scope: when to use this skill vs `skill:malloy-analysis-report`\n\nThis skill is for **curated `.malloynb` notebooks**. Interactive parameter widgets render automatically from the `given:` declarations on the models it imports (or, in older models, from deprecated `#(filter)` annotations). The notebook itself never declares parameters or filters.\n\nFor **ad-hoc reports** generated by an analysis agent, combining queries into a quick narrative artifact, use `skill:malloy-analysis-report` instead. Ad-hoc reports also inherit whatever the source declares.\n\n## Important Limitations\n\n**Compile errors in `.malloynb` files are NOT shown in the IDE linter.** You will only see errors when cells are executed. Be extra careful with syntax and test queries in the model first if unsure.\n\n## Multiple Models & Cross-Model Joins\n\nA published `.malloynb` runs each `run:` cell against the whole notebook, which compiles with **all** its `import`s in scope. So a notebook can import several `.malloy` models, and a single cell can reference, and **join**, sources from different imported files. Import each model you need in a setup cell at the top; no \"re-export\" wrapper model is required.\n\n```\n>>>malloy\nimport \"flights.malloy\"\nimport \"carriers.malloy\"\n\n>>>malloy\n// joins `flights` (from flights.malloy) to `carriers` (from carriers.malloy)\nrun: flights extend {\n join_one: carriers on carrier = carriers.code\n} -> {\n group_by: carriers.nickname\n aggregate: flight_count\n}\n```\n\nTwo things still hold:\n\n- **Imports are notebook-wide, declare them at the top, never inside a `run:` cell.** A cell's query runs as a *restricted query* against the already-compiled notebook, so its imports must already be in place. You can't `import` a model inside a query cell to scope it there; an in-query import is rejected (`file imports are not permitted in a restricted query`).\n- **Compile errors surface only at publish/render, not in the IDE linter** (see above). Verify a multi-model notebook by publishing to a scratch environment and loading it, rather than relying only on single-model query checks.\n\n## No Data-Specific Insights in Markdown\n\n**Notebooks must NOT contain findings about current data values.** Data refreshes will make these stale. Use markdown for framing questions and structural narrative, not for stating results.\n\n```\n>>>markdown\n## WRONG: will become stale when data refreshes\nRevenue spiked 23% in March. Electronics drove 60% of the spike.\n\n>>>markdown\n## RIGHT: frames the question, lets the query answer it\nHow is revenue trending? Which categories are driving changes?\n```\n\n## Cell Syntax\n\nCells delimited by `>>>markdown` or `>>>malloy`:\n\n```\n>>>markdown\n# Sales Analysis\nOverview of sales trends.\n\n>>>malloy\nimport \"sales_model.malloy\"\n\n>>>markdown\n## Summary\n\n>>>malloy\nrun: orders -> summary\n\n>>>markdown\n## Monthly Trend\n\n>>>malloy\n# line_chart\nrun: orders -> by_month\n```\n\n**NEVER create `>>>malloysql` cells.** Only use `>>>malloy` and `>>>markdown`.\n\n## Interactive Parameters (`given:`)\n\nInteractive parameters are common and important. Add them to most notebooks. **Use `given:` for anything new.** The older `#(filter)` mechanism still works and is documented below, but it is deprecated: do not reach for it just because you saw it in an existing model.\n\nA notebook's parameter surface comes from the `given:` declarations on the models it imports, not from anything declared in the `.malloynb` itself. When an imported model declares givens, Publisher's notebook UI automatically renders a Parameters panel above the notebook: declared givens become parameter inputs, the values a user sets are forwarded to Malloy's runtime, and every cell re-executes against the model with those values applied.\n\nThe widget for each parameter follows the given's declared Malloy type (`string`, `string[]`, `number`, `boolean`, `date`/`timestamp`, `filter<T>`); see `docs/givens.md` for the full type table. A `#(description=\"...\")` annotation on the given renders as helper text under its input.\n\n```malloy\n##! experimental.givens\n\n#(description=\"Two-letter IATA carrier code to spotlight\")\ngiven: carrier :: string is 'WN'\n\nsource: spotlight_carrier is duckdb.table('data/carriers.parquet') extend {\n view: by_name is {\n where: code = $carrier\n select: code, name, nickname\n limit: 1\n }\n}\n```\n\nDeclare givens at the **top of the model file**, before the source that uses them, not in the notebook. `malloy-model` § Access Control covers the syntax and the `#(authorize)`/`#(access_filter)` gating story built on top of givens.\n\n## Legacy: `#(filter)` annotations\n\n> **Deprecated. Read this before copying the example below.** `#(filter)` is server-side `where:` injection, superseded by native Malloy `given:` above. Do NOT add new `#(filter)` annotations; declare runtime parameters as givens instead. `docs/givens.md` has the migration recipes. What follows is here so you can read and maintain models that already use it.\n\nFilters are declared **once, on the model's source**. The notebook itself does not declare or list filters. Publisher reads the source's filter metadata and renders the widgets above the notebook automatically, then injects `where:` clauses server-side on every cell.\n\nDeclarations live in the `.malloy` model file, above the source, using key=value parameters. The full reference (syntax, filter types, `required` / `implicit` flags) lives with the `#(filter)` guidance in the `malloy-model` skill's Parameterizable Filters section. Short example:\n\n```malloy\n#(filter) name=Manufacturer dimension=Manufacturer type=in\n#(filter) name=OrderStatus dimension=order_status type=in\n#(filter) name=Recall_After dimension=\"Report Received Date\" type=greater_than\n#(filter) name=Recall_Before dimension=\"Report Received Date\" type=less_than\nsource: recalls is duckdb.table('data/auto_recalls.csv') extend {\n measure:\n recall_count is count()\n}\n```\n\n`#(filter)` lines go **above** the `source:` declaration and reference dimensions by name (use `dimension=`). `name` defaults to the dimension name and only needs to be set when two filters target the same dimension (e.g. a date range). `type` is the comparator: `equal`, `in`, `like`, `greater_than`, `less_than`.\n\n## Notebook Style: Iterative Analysis (Default)\n\n**Build a story.** Each query should reveal something that motivates the next query. Start broad, then drill into what's interesting. Frame questions in markdown, let queries answer them.\n\n**Pattern:** Question, Query, Next Question, Drill\n\n```\n>>>markdown\n## How is revenue trending?\n\n>>>malloy\n# line_chart\nrun: orders -> { group_by: order_month; aggregate: revenue }\n\n>>>markdown\n## What's driving the biggest changes?\n\n>>>malloy\n# bar_chart\nrun: orders -> { group_by: category; aggregate: revenue; order_by: revenue desc; limit: 10 }\n\n>>>markdown\n## How does the top category break down?\n\n>>>malloy\nrun: orders -> { aggregate: order_count, avg_order_value; where: category = 'Electronics' }\n```\n\n**When to use other styles:**\n- **Dashboard with filters:** User asks for \"filterable dashboard\". Show key KPIs and breakdowns with interactive filters.\n- **EDA/summary:** User asks for \"overview\". Run views like `summary`, `by_month`, `by_category` in sequence.\n\n## View Refinement\n\nTo add `limit`, `where`, or other options to an existing view, use `+`:\n\n```malloy\nrun: source -> my_view + { limit: 15 }\nrun: source -> my_view + { where: status = 'active', limit: 10 }\n```\n\n## Template\n\n```\n>>>markdown\n# [Title]\n[Framing question: what are we trying to understand?]\n\n>>>malloy\nimport \"model.malloy\"\n\n>>>markdown\n## [First question: start broad]\n\n>>>malloy\nrun: main_source -> [broad query]\n\n>>>markdown\n## [Next question: motivated by what the first query reveals]\n\n>>>malloy\nrun: main_source -> [drill into finding]\n\n>>>markdown\n## [Deeper question]\n\n>>>malloy\nrun: main_source -> [further breakdown]\n```\n\n## Common Mistakes\n\n| Mistake | Fix |\n|---------|-----|\n| `view_name { limit: 10 }` | Use `+`: `view_name + { limit: 10 }` |\n| `# currency` on non-money | Only use `# currency` for monetary values |\n| `#(filter) {\"type\": \"Star\"}` (JSON-blob form, on a dimension) | Unsupported legacy syntax. `#(filter)` goes **above the source** with key=value parameters (`name=`, `dimension=`, `type=`). See the `malloy-model` skill's Parameterizable Filters section. |\n| `##(filters) [...]` annotation in the notebook | Unsupported. The notebook does not declare or list filters. Publisher renders widgets automatically from the source's `#(filter)` annotations. |\n| Creating MalloySQL cells | **NEVER use `>>>malloysql`**, only `>>>malloy` and `>>>markdown` |\n| Data-specific insights in markdown | Don't write \"Revenue grew 23%.\" Frame questions instead. Data refreshes will make findings stale. |\n| `import` inside a `run:` cell (to scope a model to that cell) | Imports are notebook-wide, put them in a setup cell at the top. An in-query import is rejected: `file imports are not permitted in a restricted query` (see Multiple Models & Cross-Model Joins). |\n| Cross-model join inside a single ad-hoc report cell | A `skill:malloy-analysis-report` cell renders against a single model, so a cross-model join there won't render. Build a published `.malloynb` (which runs against the whole notebook) for a cross-model join. |\n\n## Best Practices\n\n1. Start with markdown framing the question\n2. Import each model the notebook needs in a setup cell at the top, cells can reference and join sources across all imports (see Multiple Models & Cross-Model Joins); verify a multi-model notebook by publishing to a scratch environment\n3. Each query should follow from what the previous one could reveal\n4. One query per cell\n5. Charts render only the FIRST aggregate\n6. Add interactive filters to most notebooks by declaring `#(filter)` on the model's source (see Interactive Filters), or prefer `given:` runtime parameters for new models (see Runtime Parameters)\n7. Never include data-specific findings in markdown. Frame questions, let queries answer\n\n## Done\n\nStep complete. Output: `.malloynb` notebook file."
|
|
325220
325715
|
},
|
|
325221
325716
|
{
|
|
325222
325717
|
name: "malloy-patterns",
|
|
@@ -325300,7 +325795,7 @@ Two optional fields opt the package into curation:
|
|
|
325300
325795
|
|
|
325301
325796
|
**Why curate here:** declaring \`explores\` routes agents to the well-documented curated sources instead of raw tables, and \`queryableSources: "declared"\` keeps them from reaching the hidden sources by name. The two axes compose: list a file in \`explores\` for its models to be discoverable, and \`export\` a source within that file for it to be a landing point.
|
|
325302
325797
|
|
|
325303
|
-
> **Not access control.** \`queryableSources\` gates the query surface (the query endpoints, REST and MCP alike), not compile and not raw file retrieval by exact path: \`/compile\` and \`compile_model\` are deliberately exempt, because compile is the authoring loop and the boundary is discovery curation. It doesn't restrict *who* may query, only *what* is queryable by name. Queryable sources are the union of every \`explores\`-listed file's \`export {}\` closure, whichever listed model path a query addresses them through. To gate access by caller-supplied identity/role, use \`#(authorize)\` on the source, see \`skill:malloy-model\` § Access Control and \`docs/authorize.md\`. Discovery curation and
|
|
325798
|
+
> **Not access control.** \`queryableSources\` gates the query surface (the query endpoints, REST and MCP alike), not compile and not raw file retrieval by exact path: \`/compile\` and \`compile_model\` are deliberately exempt, because compile is the authoring loop and the boundary is discovery curation. It doesn't restrict *who* may query, only *what* is queryable by name. Queryable sources are the union of every \`explores\`-listed file's \`export {}\` closure, whichever listed model path a query addresses them through. To gate access by caller-supplied identity/role, use \`#(authorize)\` on the source (and \`#(access_filter)\` to scope rows), see \`skill:malloy-model\` § Access Control and \`docs/authorize.md\`. Discovery curation and these gates are independent layers.
|
|
325304
325799
|
|
|
325305
325800
|
The manifest also carries a \`scope\` field (\`"package"\` | \`"version"\`, default \`"package"\`) controlling whether persisted/materialized artifacts are shared across published versions or owned by a single version, and a \`materialization\` field configuring that persistence policy (a cron \`schedule\` or a \`freshness\` window). Both are unrelated to discovery curation; there is no per-source \`sharing\` or \`schedule\` field, that was retired in favor of the single package-level \`scope\` and \`materialization\`.
|
|
325306
325801
|
|
|
@@ -325433,10 +325928,10 @@ Wrong: \`pick('Premium') { ... }\` (that's not Malloy syntax).
|
|
|
325433
325928
|
**Window functions with \`calculate:\`:** running totals, \`lag()\`, \`lead()\`, and other window operations belong in \`calculate:\`, not \`aggregate:\`.
|
|
325434
325929
|
\`\`\`malloy
|
|
325435
325930
|
run: source -> {
|
|
325436
|
-
group_by:
|
|
325931
|
+
group_by: order_month is order_date.month
|
|
325437
325932
|
aggregate: revenue
|
|
325438
325933
|
calculate: prev_month_revenue is lag(revenue)
|
|
325439
|
-
order_by:
|
|
325934
|
+
order_by: order_month
|
|
325440
325935
|
}
|
|
325441
325936
|
\`\`\`
|
|
325442
325937
|
|
|
@@ -325474,15 +325969,15 @@ where: order_date >= @2020-01-01
|
|
|
325474
325969
|
|
|
325475
325970
|
### Truncation accessors return timestamps, not integers
|
|
325476
325971
|
|
|
325477
|
-
\`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer
|
|
325972
|
+
\`order_date.month\` returns a month-truncated timestamp (e.g., \`2025-06-01 00:00:00\`), useful for \`group_by:\`. It is NOT a 1-12 integer.
|
|
325478
325973
|
|
|
325479
|
-
Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`.
|
|
325974
|
+
Available truncations: \`.year\`, \`.quarter\`, \`.month\`, \`.week\`, \`.day\`, \`.hour\`, \`.minute\`, \`.second\`.
|
|
325480
325975
|
|
|
325481
|
-
### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved
|
|
325976
|
+
### \`month\`, \`year\`, \`day\`, \`quarter\` are reserved as names
|
|
325482
325977
|
|
|
325483
|
-
|
|
325978
|
+
They cannot name an output field. The functions are fine: \`month(order_date)\` extracts 1-12 anywhere, including \`where:\`.
|
|
325484
325979
|
|
|
325485
|
-
Wrong: \`group_by: month is order_date.month\` →
|
|
325980
|
+
Wrong: \`group_by: month is order_date.month\` → \`'month' is a reserved word, so to use it as a name you must quote it\`
|
|
325486
325981
|
Right: \`group_by: order_month is order_date.month\`
|
|
325487
325982
|
Right (when a column is literally named \`month\`): \`\` group_by: \`month\` \`\`
|
|
325488
325983
|
|
|
@@ -325598,7 +326093,7 @@ run: source -> {
|
|
|
325598
326093
|
}
|
|
325599
326094
|
\`\`\`
|
|
325600
326095
|
|
|
325601
|
-
Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A
|
|
326096
|
+
Charts render only the **first** aggregate. For multiple measures on one chart, place \`# y\` above the \`aggregate:\` keyword or use the \`y=['a','b']\` shorthand. A chart annotation left as the last line inside \`{ }\` fails with *"Parser enountered unexpected statement type 'unimplemented'"* (the compiler's spelling); one after the closing \`}\` fails with *"Object annotation not connected to any object"*.
|
|
325602
326097
|
|
|
325603
326098
|
Read the \`malloy-charts\` skill for chart types, properties, data shape requirements, and selection guidance.
|
|
325604
326099
|
|
|
@@ -325609,11 +326104,10 @@ Read the error against the tables above and below. Most failures match a known p
|
|
|
325609
326104
|
| Error message | Likely cause / fix |
|
|
325610
326105
|
|---|---|
|
|
325611
326106
|
| \`Cannot compare a timestamp to a number\` | Comparing \`date.year\` to an integer. Use \`date >= @2020-01-01\` instead. |
|
|
325612
|
-
| \`no viable alternative at input '<word>'\` |
|
|
326107
|
+
| \`no viable alternative at input '<word>'\` | Often a \`;\` between fields within one clause - fields under one \`aggregate:\`/\`group_by:\` are comma- or newline-separated (the error points at the field right after the \`;\`). |
|
|
325613
326108
|
| \`'<field_name>' is not defined\` | Field doesn't exist in the source. Re-check against the model definition; you may have stripped a join prefix. |
|
|
325614
|
-
| \`missing {DAY, HOUR, MINUTE, MONTH, QUARTER, SECOND, WEEK, YEAR}\` | Chained date property too deep (e.g., \`.month.something\`). Stop at the first truncation. |
|
|
325615
326109
|
| \`field is a bar chart, but is not a repeated record\` | Chart annotation placed inside \`{ }\`. Move \`# bar_chart\` above \`run:\` / \`view:\` / \`nest:\`. |
|
|
325616
|
-
| \`Parser
|
|
326110
|
+
| \`Parser enountered unexpected statement\` | Spelled that way by the compiler. Most often a chart annotation left as the last line inside \`{ }\` - move it above \`run:\`. Also syntax Malloy doesn't allow in that position (e.g., \`pick\` inside a nested view). |
|
|
325617
326111
|
| Query silently returns zero rows | Filter value mismatch (case, spelling, format). Run a distinct-values query on the dimension to confirm the literal. |
|
|
325618
326112
|
|
|
325619
326113
|
## Syntax Help
|
|
@@ -327699,7 +328193,7 @@ async function withBigQueryLookupRetry(work, parentJobId) {
|
|
|
327699
328193
|
parentJobId,
|
|
327700
328194
|
error: error instanceof Error ? error.message : String(error)
|
|
327701
328195
|
});
|
|
327702
|
-
await new Promise((
|
|
328196
|
+
await new Promise((resolve7) => setTimeout(resolve7, BIGQUERY_LOOKUP_BACKOFF_MS * attempt));
|
|
327703
328197
|
}
|
|
327704
328198
|
}
|
|
327705
328199
|
}
|
|
@@ -330384,11 +330878,11 @@ var promisifyStore = (passedStore) => {
|
|
|
330384
330878
|
|
|
330385
330879
|
class PromisifiedStore {
|
|
330386
330880
|
async increment(key) {
|
|
330387
|
-
return new Promise((
|
|
330881
|
+
return new Promise((resolve7, reject2) => {
|
|
330388
330882
|
legacyStore.incr(key, (error, totalHits, resetTime) => {
|
|
330389
330883
|
if (error)
|
|
330390
330884
|
reject2(error);
|
|
330391
|
-
|
|
330885
|
+
resolve7({ totalHits, resetTime });
|
|
330392
330886
|
});
|
|
330393
330887
|
});
|
|
330394
330888
|
}
|
|
@@ -330518,9 +331012,9 @@ var rateLimit = (passedOptions) => {
|
|
|
330518
331012
|
}
|
|
330519
331013
|
}
|
|
330520
331014
|
const middleware = handleAsyncErrors(async (request, response, next) => {
|
|
330521
|
-
const closePromise = config.skipFailedRequests && new Promise((
|
|
330522
|
-
const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((
|
|
330523
|
-
const errorPromise = config.skipFailedRequests && new Promise((
|
|
331015
|
+
const closePromise = config.skipFailedRequests && new Promise((resolve7) => response.once("close", resolve7));
|
|
331016
|
+
const finishPromise = (config.skipFailedRequests || config.skipSuccessfulRequests) && new Promise((resolve7) => response.once("finish", resolve7));
|
|
331017
|
+
const errorPromise = config.skipFailedRequests && new Promise((resolve7) => response.once("error", resolve7));
|
|
330524
331018
|
debug2("requested %o", request.originalUrl);
|
|
330525
331019
|
debug2("request from ip %o", request.ip);
|
|
330526
331020
|
const skip = await config.skip(request, response);
|
|
@@ -331862,8 +332356,8 @@ mainServer.timeout = 600000;
|
|
|
331862
332356
|
mainServer.keepAliveTimeout = 600000;
|
|
331863
332357
|
mainServer.headersTimeout = 600000;
|
|
331864
332358
|
var resolveRestBound = () => {};
|
|
331865
|
-
var restBound = new Promise((
|
|
331866
|
-
resolveRestBound =
|
|
332359
|
+
var restBound = new Promise((resolve8) => {
|
|
332360
|
+
resolveRestBound = resolve8;
|
|
331867
332361
|
});
|
|
331868
332362
|
mainServer.listen(PUBLISHER_PORT, PUBLISHER_HOST, async () => {
|
|
331869
332363
|
resolveRestBound();
|