@malloy-publisher/server 0.0.220 → 0.0.222
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +51 -1
- package/dist/app/assets/{EnvironmentPage-BqLiaatL.js → EnvironmentPage-D6G5n6mY.js} +1 -1
- package/dist/app/assets/{HomePage-G_xpt9XE.js → HomePage-DgZluD8u.js} +1 -1
- package/dist/app/assets/{LightMode-s1PDRIsF.js → LightMode-w9v0pEbg.js} +1 -1
- package/dist/app/assets/{MainPage-C6TfpC92.js → MainPage-Bi_Tukvr.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-BMO1afhm.js → MaterializationsPage-BVERjnmG.js} +1 -1
- package/dist/app/assets/{ModelPage-zHOJwMnU.js → ModelPage-2OjS259-.js} +1 -1
- package/dist/app/assets/{PackagePage-GCaWmELQ.js → PackagePage-C8L2On4z.js} +1 -1
- package/dist/app/assets/{RouteError-CIC588k4.js → RouteError-DJoNC_Vl.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-BdbIfIGO.js → ThemeEditorPage-B-hJ1zgd.js} +1 -1
- package/dist/app/assets/{WorkbookPage-C8ucK_H8.js → WorkbookPage-CTbDgGeJ.js} +1 -1
- package/dist/app/assets/{core-DFvqRVqM.es-NtfPVC7h.js → core-D9Hl0IDY.es-CmgzHn4c.js} +1 -1
- package/dist/app/assets/{index-aYtt-ovi.js → index-BEjJCJjX.js} +1 -1
- package/dist/app/assets/{index-aYNf0kTZ.js → index-BF8PkFm8.js} +1 -1
- package/dist/app/assets/{index-BxDMCn3s.js → index-Cr-asgoV.js} +4 -4
- package/dist/app/assets/{index-B3NcDPbp.js → index-DcCw_qxr.js} +1 -1
- package/dist/app/index.html +1 -1
- package/dist/package_load_worker.mjs +20 -2
- package/dist/server.mjs +35 -5
- package/package.json +1 -1
- package/src/controller/package.controller.spec.ts +89 -1
- package/src/controller/package.controller.ts +28 -3
- package/src/package_load/package_load_pool.ts +2 -1
- package/src/package_load/package_load_worker.ts +5 -2
- package/src/package_load/protocol.ts +2 -1
- package/src/service/build_plan.spec.ts +44 -0
- package/src/service/build_plan.ts +10 -1
- package/src/service/materialization_cron_gate.spec.ts +137 -0
- package/src/service/materialization_schedule_surface.spec.ts +30 -1
- package/src/service/materialization_test_fixtures.ts +17 -1
- package/src/service/package.ts +46 -0
- package/src/service/package_manifest.spec.ts +42 -4
- package/src/service/package_manifest.ts +53 -4
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import{aI as ci}from"./index-
|
|
1
|
+
import{aI as ci}from"./index-Cr-asgoV.js";import{r as di}from"./index-w_0OQJgZ.js";function pi(u,re){for(var j=0;j<re.length;j++){const N=re[j];if(typeof N!="string"&&!Array.isArray(N)){for(const E in N)if(E!=="default"&&!(E in u)){const S=Object.getOwnPropertyDescriptor(N,E);S&&Object.defineProperty(u,E,S.get?S:{enumerable:!0,get:()=>N[E]})}}}return Object.freeze(Object.defineProperty(u,Symbol.toStringTag,{value:"Module"}))}var bt={},ut={},Or;function Vt(){if(Or)return ut;Or=1,Object.defineProperty(ut,"__esModule",{value:!0}),ut.isChainOp=u,ut.isStringCondition=re,ut.isStringFilter=j,ut.isBooleanFilter=N,ut.isNumberFilter=E,ut.isTemporalFilter=S,ut.isFilterExpression=w,ut.isFilterable=y;function u(o){return["and","or",","].includes(o)}function re(o){return["starts","ends","contains","="].includes(o.operator)}function j(o){return typeof o=="object"&&o!==null&&"operator"in o&&typeof o.operator=="string"&&["starts","ends","contains","=","~","null","empty","and","or",",","()"].includes(o.operator)}function N(o){return typeof o=="object"&&o!==null&&"operator"in o&&typeof o.operator=="string"&&["null","true","false","=false","=true"].includes(o.operator)}function E(o){return typeof o=="object"&&o!==null&&"operator"in o&&typeof o.operator=="string"&&["range","<=",">=","!=","=",">","<","and","or","()","null"].includes(o.operator)}function S(o){return typeof o=="object"&&o!==null&&"operator"in o&&typeof o.operator=="string"&&["literal","before","after","to","for","in","and","or","in_last","this","last","next","()","null"].includes(o.operator)}function w(o){return typeof o=="object"&&o!==null&&"operator"in o}function y(o){return["string","number","boolean","timestamp","timestamptz","date"].includes(o)}return ut}var Pt={},Er;function hi(){return Er||(Er=1,Object.defineProperty(Pt,"__esModule",{value:!0}),Pt.BooleanFilterExpression=void 0,Pt.BooleanFilterExpression={parse(u){var re;if(u.match(/^\s*$/))return{parsed:null,log:[]};const j={parsed:null,log:[]};let N=u.toLowerCase().trim().replace(/\s\s+/," "),E=!1;if(N.startsWith("not ")&&(E=!0,N=N.slice(4)),N==="true")j.parsed={operator:"true"};else if(N==="=true")j.parsed={operator:"=true"};else if(N==="false")j.parsed={operator:"false"};else if(N==="=false")j.parsed={operator:"=false"};else if(N==="null")j.parsed={operator:"null"};else{const S=u.match(/[^\s]/),w=S&&(re=S.index)!==null&&re!==void 0?re:0;j.log=[{message:`Illegal boolean filter '${N}'. Must be one of true,=true,false,=false,null`,severity:"error",startIndex:w,endIndex:w+u.length-1}]}return E&&j.parsed&&(j.parsed.not=!0),j},unparse(u){return u===null?"":(u.not?"not ":"")+u.operator}}),Pt}var pr={},Ue={},kr;function Ht(){if(kr)return Ue;kr=1,Object.defineProperty(Ue,"__esModule",{value:!0}),Ue.maybeNot=re,Ue.unescape=j,Ue.escape=N,Ue.matchOp=S,Ue.conjoin=y,Ue.joinNumbers=o,Ue.mkRange=d,Ue.mkValues=t,Ue.numNot=D,Ue.temporalNot=x,Ue.joinTemporal=Z,Ue.timeLiteral=ee,Ue.mkUnits=ce;const u=Vt();function re(O){const[A,b]=O;return A&&b&&(0,u.isStringFilter)(b)?{...b,not:!0}:b}function j(O){return O.replace(/\\(.)/g,"$1")}function N(O){const A=O.toLowerCase();return A==="null"||A==="empty"?"\\"+O:O.replace(/([,; |()\\%_-])/g,"\\$1")}function E(O){let A=!1,b=!1,B=0,de=!1;const Fe=O.length;for(let ve=0;ve<Fe;ve+=1){const Ye=O[ve];if(Ye===" "||Ye===" "){B+=1;continue}B=0,Ye==="%"?(de=!0,ve===0&&(A=!0),b=!0):(b=!1,Ye==="\\"?ve+=1:Ye==="_"&&(de=!0))}return{hasLike:de,percentEnd:b&&Fe>1,percentStart:A&&Fe>1,endSpace:B}}function S(O){let A=O.trimStart();const{hasLike:b,percentEnd:B,percentStart:de,endSpace:Fe}=E(A);if(Fe>0&&(A=A.slice(0,-Fe)),b){if(de&&B){const ve=A.slice(1,-1);if(!E(ve).hasLike&&ve.length>0)return{operator:"contains",values:[j(ve)]}}else if(B){const ve=A.slice(0,-1);if(!E(ve).hasLike)return{operator:"starts",values:[j(ve)]}}else if(de){const ve=A.slice(1);if(!E(ve).hasLike)return{operator:"ends",values:[j(ve)]}}return{operator:"~",escaped_values:[A]}}return A.toLowerCase()==="null"?{operator:"null"}:A.toLowerCase()==="empty"?{operator:"empty"}:{operator:"=",values:[j(A)]}}function w(O,A){var b,B;return O.operator===A.operator&&((b=O.not)!==null&&b!==void 0?b:!1)===((B=A.not)!==null&&B!==void 0?B:!1)}function y(O,A,b){if(A=A.toLowerCase(),(0,u.isStringFilter)(O)&&(0,u.isStringFilter)(b)){if(A===","){if(O.operator==="~"&&w(O,b))return{...O,escaped_values:[...O.escaped_values,...b.escaped_values]};if((0,u.isStringCondition)(O)&&w(O,b))return{...O,values:[...O.values,...b.values]}}const B=A===","?",":A==="|"?"or":A===";"?"and":void 0;if(B)return O.operator===B?{...O,members:[...O.members,b]}:{operator:B,members:[O,b]}}return null}function o(O,A,b){if(A=A.toLowerCase(),(0,u.isNumberFilter)(O)&&(0,u.isNumberFilter)(b)){if(A==="or"&&O.operator==="="&&w(O,b)){const B={operator:"=",values:[...O.values,...b.values]};return O.not&&(B.not=!0),B}if(A==="and"||A==="or")return O.operator===A?{...O,members:[...O.members,b]}:{operator:A,members:[O,b]}}return null}function d(O,A,b,B){return{operator:"range",startValue:A,startOperator:O==="("?">":">=",endValue:b,endOperator:B===")"?"<":"<="}}function t(O,A){return{values:[O,...A]}}function D(O,A){return(0,u.isNumberFilter)(O)&&A?O.operator==="="?{operator:"!=",values:O.values}:O.operator==="!="?{operator:"=",values:O.values}:{...O,not:!0}:O}function x(O,A){if((0,u.isTemporalFilter)(O)&&A){if("not"in O){const b={...O};return O.not?delete b.not:b.not=!0,b}return{...O,not:!0}}return O}function Z(O,A,b){return A=A.toLowerCase(),(0,u.isTemporalFilter)(O)&&(0,u.isTemporalFilter)(b)&&(A==="and"||A==="or")?O.operator===A?{...O,members:[...O.members,b]}:{operator:A,members:[O,b]}:null}function ee(O,A){const b={moment:"literal",literal:O};return A&&(b.units=A),b}function ce(O){switch(O.toLowerCase()){case"second":case"seconds":return"second";case"minute":case"minutes":return"minute";case"hour":case"hours":return"hour";case"day":case"days":return"day";case"week":case"weeks":return"week";case"month":case"months":return"month";case"quarter":case"quarters":return"quarter";case"year":case"years":return"year"}}return Ue}var hr,Rr;function mi(){if(Rr)return hr;Rr=1;const{numNot:u,mkRange:re,joinNumbers:j,mkValues:N}=Ht();function E(o,d){function t(){this.constructor=o}t.prototype=d.prototype,o.prototype=new t}function S(o,d,t,D){var x=Error.call(this,o);return Object.setPrototypeOf&&Object.setPrototypeOf(x,S.prototype),x.expected=d,x.found=t,x.location=D,x.name="SyntaxError",x}E(S,Error);function w(o,d,t){return t=t||" ",o.length>d?o:(d-=o.length,t+=t.repeat(d),o+t.slice(0,d))}S.prototype.format=function(o){var d="Error: "+this.message;if(this.location){var t=null,D;for(D=0;D<o.length;D++)if(o[D].source===this.location.source){t=o[D].text.split(/\r\n|\n|\r/g);break}var x=this.location.start,Z=this.location.source&&typeof this.location.source.offset=="function"?this.location.source.offset(x):x,ee=this.location.source+":"+Z.line+":"+Z.column;if(t){var ce=this.location.end,O=w("",Z.line.toString().length," "),A=t[x.line-1],b=x.line===ce.line?ce.column:A.length+1,B=b-x.column||1;d+=`
|
|
2
2
|
--> `+ee+`
|
|
3
3
|
`+O+` |
|
|
4
4
|
`+Z.line+" | "+A+`
|
package/dist/app/index.html
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600;700&family=JetBrains+Mono:wght@400;500&display=swap"
|
|
12
12
|
/>
|
|
13
13
|
<title>Malloy Publisher</title>
|
|
14
|
-
<script type="module" crossorigin src="/assets/index-
|
|
14
|
+
<script type="module" crossorigin src="/assets/index-Cr-asgoV.js"></script>
|
|
15
15
|
<link rel="stylesheet" crossorigin href="/assets/index-5eLCcNmP.css">
|
|
16
16
|
</head>
|
|
17
17
|
<body>
|
|
@@ -11713,12 +11713,30 @@ function unwrapQuotedExpression(body) {
|
|
|
11713
11713
|
}
|
|
11714
11714
|
|
|
11715
11715
|
// src/service/package_manifest.ts
|
|
11716
|
+
var FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"];
|
|
11717
|
+
function parseFreshness(raw) {
|
|
11718
|
+
if (!raw || typeof raw !== "object") {
|
|
11719
|
+
return null;
|
|
11720
|
+
}
|
|
11721
|
+
const { window: window2, fallback } = raw;
|
|
11722
|
+
const freshness = {};
|
|
11723
|
+
if (typeof window2 === "string") {
|
|
11724
|
+
freshness.window = window2;
|
|
11725
|
+
}
|
|
11726
|
+
if (typeof fallback === "string" && FRESHNESS_FALLBACKS.includes(fallback)) {
|
|
11727
|
+
freshness.fallback = fallback;
|
|
11728
|
+
}
|
|
11729
|
+
return freshness;
|
|
11730
|
+
}
|
|
11716
11731
|
function parsePackageMaterialization(raw) {
|
|
11717
11732
|
if (!raw || typeof raw !== "object") {
|
|
11718
11733
|
return null;
|
|
11719
11734
|
}
|
|
11720
|
-
const schedule = raw
|
|
11721
|
-
return {
|
|
11735
|
+
const { schedule, freshness } = raw;
|
|
11736
|
+
return {
|
|
11737
|
+
schedule: typeof schedule === "string" ? schedule : null,
|
|
11738
|
+
freshness: parseFreshness(freshness)
|
|
11739
|
+
};
|
|
11722
11740
|
}
|
|
11723
11741
|
|
|
11724
11742
|
// src/service/source_extraction.ts
|
package/dist/server.mjs
CHANGED
|
@@ -249957,6 +249957,14 @@ class ModelController {
|
|
|
249957
249957
|
// src/controller/package.controller.ts
|
|
249958
249958
|
init_constants();
|
|
249959
249959
|
init_errors();
|
|
249960
|
+
function formatPublishRejections(pkg, exploresOverride) {
|
|
249961
|
+
const message = [
|
|
249962
|
+
pkg.formatInvalidExplores(exploresOverride),
|
|
249963
|
+
pkg.formatInvalidSchedule()
|
|
249964
|
+
].filter(Boolean).join(`
|
|
249965
|
+
`);
|
|
249966
|
+
return message || undefined;
|
|
249967
|
+
}
|
|
249960
249968
|
|
|
249961
249969
|
class PackageController {
|
|
249962
249970
|
environmentStore;
|
|
@@ -249997,7 +250005,7 @@ class PackageController {
|
|
|
249997
250005
|
let result;
|
|
249998
250006
|
if (body.location) {
|
|
249999
250007
|
const bodyLocation = body.location;
|
|
250000
|
-
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg
|
|
250008
|
+
result = await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => formatPublishRejections(pkg));
|
|
250001
250009
|
} else {
|
|
250002
250010
|
result = await environment.addPackage(packageName);
|
|
250003
250011
|
}
|
|
@@ -250005,7 +250013,7 @@ class PackageController {
|
|
|
250005
250013
|
throw new Error(`Failed to create package ${packageName}`);
|
|
250006
250014
|
}
|
|
250007
250015
|
if (!body.location) {
|
|
250008
|
-
const invalidMsg = result
|
|
250016
|
+
const invalidMsg = formatPublishRejections(result);
|
|
250009
250017
|
if (invalidMsg) {
|
|
250010
250018
|
await environment.unloadPackage(packageName).catch(() => {});
|
|
250011
250019
|
throw new BadRequestError(invalidMsg);
|
|
@@ -250030,7 +250038,7 @@ class PackageController {
|
|
|
250030
250038
|
const environment = await this.environmentStore.getEnvironment(environmentName, false);
|
|
250031
250039
|
if (body.location) {
|
|
250032
250040
|
const bodyLocation = body.location;
|
|
250033
|
-
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => pkg
|
|
250041
|
+
await environment.installPackage(packageName, (stagingPath) => this.downloadInto(environmentName, packageName, bodyLocation, stagingPath), (pkg) => formatPublishRejections(pkg, body.explores?.map(normalizeModelPath)));
|
|
250034
250042
|
}
|
|
250035
250043
|
const result = await environment.updatePackage(packageName, body);
|
|
250036
250044
|
await this.environmentStore.addPackageToDatabase(environmentName, packageName);
|
|
@@ -258525,6 +258533,7 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
|
|
|
258525
258533
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
258526
258534
|
if (include && !include.has(source.name))
|
|
258527
258535
|
continue;
|
|
258536
|
+
const annotationFields = deriveAnnotationFields(source);
|
|
258528
258537
|
wireSources[sourceID] = {
|
|
258529
258538
|
name: source.name,
|
|
258530
258539
|
sourceID: source.sourceID,
|
|
@@ -258532,8 +258541,10 @@ function deriveBuildPlan(graphs, sources, connectionDigests, sourceNames, source
|
|
|
258532
258541
|
dialect: source.dialectName,
|
|
258533
258542
|
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
258534
258543
|
sql: source.getSQL(),
|
|
258544
|
+
sharing: annotationFields.sharing ?? null,
|
|
258545
|
+
refresh: annotationFields.refresh ?? null,
|
|
258535
258546
|
columns: deriveColumns(source),
|
|
258536
|
-
annotationFields
|
|
258547
|
+
annotationFields,
|
|
258537
258548
|
modelPath: sourceModelPaths?.[sourceID]
|
|
258538
258549
|
};
|
|
258539
258550
|
}
|
|
@@ -258699,7 +258710,8 @@ class Package {
|
|
|
258699
258710
|
queryableSources: outcome.packageMetadata.queryableSources,
|
|
258700
258711
|
manifestLocation: outcome.packageMetadata.manifestLocation ?? null,
|
|
258701
258712
|
materialization: outcome.packageMetadata.materialization ?? {
|
|
258702
|
-
schedule: null
|
|
258713
|
+
schedule: null,
|
|
258714
|
+
freshness: null
|
|
258703
258715
|
}
|
|
258704
258716
|
};
|
|
258705
258717
|
const models = new Map;
|
|
@@ -258753,6 +258765,13 @@ class Package {
|
|
|
258753
258765
|
detail: invalidMsg
|
|
258754
258766
|
});
|
|
258755
258767
|
}
|
|
258768
|
+
const invalidSchedule = pkg.formatInvalidSchedule();
|
|
258769
|
+
if (invalidSchedule) {
|
|
258770
|
+
logger.warn(`Package ${packageName} has an invalid cron schedule`, {
|
|
258771
|
+
packageName,
|
|
258772
|
+
detail: invalidSchedule
|
|
258773
|
+
});
|
|
258774
|
+
}
|
|
258756
258775
|
pkg.logEmptyDiscoveryWarnings();
|
|
258757
258776
|
return pkg;
|
|
258758
258777
|
}
|
|
@@ -258824,6 +258843,17 @@ class Package {
|
|
|
258824
258843
|
}
|
|
258825
258844
|
formatInvalidExplores(exploresOverride) {
|
|
258826
258845
|
return this.exploreWarnings(exploresOverride).join(`
|
|
258846
|
+
`);
|
|
258847
|
+
}
|
|
258848
|
+
scheduleWarnings() {
|
|
258849
|
+
const schedule = this.packageMetadata.materialization?.schedule;
|
|
258850
|
+
if (!schedule)
|
|
258851
|
+
return [];
|
|
258852
|
+
const sources = Object.values(this.buildPlan?.sources ?? {});
|
|
258853
|
+
return sources.filter((source) => source.sharing !== "private").map((source) => `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` + `persist source to declare '#@ persist ... sharing=private'; source ` + `'${source.name}' resolves to ${source.sharing ? `'${source.sharing}'` : "unset"}. Declare 'materialization.freshness.window' instead (the control plane ` + `schedules refreshes from it), or mark every persist source sharing=private.`);
|
|
258854
|
+
}
|
|
258855
|
+
formatInvalidSchedule() {
|
|
258856
|
+
return this.scheduleWarnings().join(`
|
|
258827
258857
|
`);
|
|
258828
258858
|
}
|
|
258829
258859
|
emptyDiscoveryWarnings() {
|
package/package.json
CHANGED
|
@@ -18,6 +18,7 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
18
18
|
"Invalid explores entry 'missing.malloy' in publisher.json: file not found";
|
|
19
19
|
const mockPackage = {
|
|
20
20
|
formatInvalidExplores: () => invalidMsg,
|
|
21
|
+
formatInvalidSchedule: () => "",
|
|
21
22
|
};
|
|
22
23
|
const unloadPackage = sinon.stub().resolves(undefined);
|
|
23
24
|
const deletePackage = sinon.stub().resolves(undefined);
|
|
@@ -52,7 +53,10 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
52
53
|
// controller passes a validator and does NOT call delete/unload itself.
|
|
53
54
|
const invalidMsg =
|
|
54
55
|
"Invalid explores entry 'missing.malloy' in publisher.json: file not found";
|
|
55
|
-
const mockPackage = {
|
|
56
|
+
const mockPackage = {
|
|
57
|
+
formatInvalidExplores: () => invalidMsg,
|
|
58
|
+
formatInvalidSchedule: () => "",
|
|
59
|
+
};
|
|
56
60
|
// installPackage mimics the real contract: invoke the validator and, if it
|
|
57
61
|
// returns a message, throw BadRequestError (after its internal rollback).
|
|
58
62
|
const installPackage = sinon
|
|
@@ -100,6 +104,7 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
100
104
|
it("persists when explores are valid (no-location)", async () => {
|
|
101
105
|
const mockPackage = {
|
|
102
106
|
formatInvalidExplores: () => "",
|
|
107
|
+
formatInvalidSchedule: () => "",
|
|
103
108
|
};
|
|
104
109
|
const addPackage = sinon.stub().resolves(mockPackage);
|
|
105
110
|
const getEnvironment = sinon.stub().resolves({ addPackage });
|
|
@@ -122,6 +127,88 @@ describe("PackageController.addPackage explores validation", () => {
|
|
|
122
127
|
});
|
|
123
128
|
});
|
|
124
129
|
|
|
130
|
+
describe("PackageController.addPackage cron schedule validation", () => {
|
|
131
|
+
afterEach(() => {
|
|
132
|
+
sinon.restore();
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
it("rejects a publish whose cron fails the private-only gate (no-location path)", async () => {
|
|
136
|
+
// Valid explores but an invalid materialization cron: the publish must
|
|
137
|
+
// still 400 (strict-at-publish, same split as explores — load merely
|
|
138
|
+
// warns) and roll back via unloadPackage.
|
|
139
|
+
const cronMsg =
|
|
140
|
+
"materialization.schedule (cron) in publisher.json requires every " +
|
|
141
|
+
"persist source to declare '#@ persist ... sharing=private'; source " +
|
|
142
|
+
"'orders' resolves to unset.";
|
|
143
|
+
const mockPackage = {
|
|
144
|
+
formatInvalidExplores: () => "",
|
|
145
|
+
formatInvalidSchedule: () => cronMsg,
|
|
146
|
+
};
|
|
147
|
+
const unloadPackage = sinon.stub().resolves(undefined);
|
|
148
|
+
const addPackage = sinon.stub().resolves(mockPackage);
|
|
149
|
+
const environment = { addPackage, unloadPackage };
|
|
150
|
+
const getEnvironment = sinon.stub().resolves(environment);
|
|
151
|
+
const addPackageToDatabase = sinon.stub().resolves(undefined);
|
|
152
|
+
const environmentStore = {
|
|
153
|
+
publisherConfigIsFrozen: false,
|
|
154
|
+
getEnvironment,
|
|
155
|
+
addPackageToDatabase,
|
|
156
|
+
} as unknown as EnvironmentStore;
|
|
157
|
+
|
|
158
|
+
const controller = new PackageController(environmentStore);
|
|
159
|
+
|
|
160
|
+
await expect(
|
|
161
|
+
controller.addPackage("env", { name: "pkg", description: "test" }),
|
|
162
|
+
).rejects.toThrow(cronMsg);
|
|
163
|
+
|
|
164
|
+
expect(unloadPackage.calledOnceWith("pkg")).toBe(true);
|
|
165
|
+
expect(addPackageToDatabase.called).toBe(false);
|
|
166
|
+
});
|
|
167
|
+
|
|
168
|
+
it("location path: the cron gate runs inside installPackage's rollback window", async () => {
|
|
169
|
+
const cronMsg =
|
|
170
|
+
"materialization.schedule (cron) in publisher.json requires every " +
|
|
171
|
+
"persist source to declare '#@ persist ... sharing=private'.";
|
|
172
|
+
const mockPackage = {
|
|
173
|
+
formatInvalidExplores: () => "",
|
|
174
|
+
formatInvalidSchedule: () => cronMsg,
|
|
175
|
+
};
|
|
176
|
+
const installPackage = sinon
|
|
177
|
+
.stub()
|
|
178
|
+
.callsFake(
|
|
179
|
+
async (
|
|
180
|
+
_name: string,
|
|
181
|
+
_downloader: unknown,
|
|
182
|
+
validate?: (pkg: unknown) => string | undefined,
|
|
183
|
+
) => {
|
|
184
|
+
const msg = validate?.(mockPackage);
|
|
185
|
+
if (msg) throw new BadRequestError(msg);
|
|
186
|
+
return mockPackage;
|
|
187
|
+
},
|
|
188
|
+
);
|
|
189
|
+
const environment = { installPackage };
|
|
190
|
+
const getEnvironment = sinon.stub().resolves(environment);
|
|
191
|
+
const addPackageToDatabase = sinon.stub().resolves(undefined);
|
|
192
|
+
const environmentStore = {
|
|
193
|
+
publisherConfigIsFrozen: false,
|
|
194
|
+
getEnvironment,
|
|
195
|
+
addPackageToDatabase,
|
|
196
|
+
} as unknown as EnvironmentStore;
|
|
197
|
+
|
|
198
|
+
const controller = new PackageController(environmentStore);
|
|
199
|
+
|
|
200
|
+
await expect(
|
|
201
|
+
controller.addPackage("env", {
|
|
202
|
+
name: "pkg",
|
|
203
|
+
description: "test",
|
|
204
|
+
location: "gs://bucket/pkg.zip",
|
|
205
|
+
}),
|
|
206
|
+
).rejects.toThrow(cronMsg);
|
|
207
|
+
|
|
208
|
+
expect(addPackageToDatabase.called).toBe(false);
|
|
209
|
+
});
|
|
210
|
+
});
|
|
211
|
+
|
|
125
212
|
describe("PackageController.updatePackage explores validation", () => {
|
|
126
213
|
afterEach(() => {
|
|
127
214
|
sinon.restore();
|
|
@@ -138,6 +225,7 @@ describe("PackageController.updatePackage explores validation", () => {
|
|
|
138
225
|
const mockPackage = {
|
|
139
226
|
formatInvalidExplores: (override?: string[]) =>
|
|
140
227
|
override?.includes("nope.malloy") ? invalidMsg : "",
|
|
228
|
+
formatInvalidSchedule: () => "",
|
|
141
229
|
};
|
|
142
230
|
const installPackage = sinon
|
|
143
231
|
.stub()
|
|
@@ -5,6 +5,30 @@ import { EnvironmentStore } from "../service/environment_store";
|
|
|
5
5
|
|
|
6
6
|
type ApiPackage = components["schemas"]["Package"];
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* Everything that is strict-at-publish, joined into one 400 message (or
|
|
10
|
+
* undefined when the package is publishable): invalid explores entries plus
|
|
11
|
+
* the materialization cron gate (a package-level `schedule` requires every
|
|
12
|
+
* persist source to resolve to explicit `sharing=private` — see
|
|
13
|
+
* Package.scheduleWarnings). At startup/reload both are warn-only instead
|
|
14
|
+
* (fail-safe; see Package.loadViaWorker).
|
|
15
|
+
*/
|
|
16
|
+
function formatPublishRejections(
|
|
17
|
+
pkg: {
|
|
18
|
+
formatInvalidExplores(exploresOverride?: string[]): string;
|
|
19
|
+
formatInvalidSchedule(): string;
|
|
20
|
+
},
|
|
21
|
+
exploresOverride?: string[],
|
|
22
|
+
): string | undefined {
|
|
23
|
+
const message = [
|
|
24
|
+
pkg.formatInvalidExplores(exploresOverride),
|
|
25
|
+
pkg.formatInvalidSchedule(),
|
|
26
|
+
]
|
|
27
|
+
.filter(Boolean)
|
|
28
|
+
.join("\n");
|
|
29
|
+
return message || undefined;
|
|
30
|
+
}
|
|
31
|
+
|
|
8
32
|
export class PackageController {
|
|
9
33
|
private environmentStore: EnvironmentStore;
|
|
10
34
|
|
|
@@ -100,7 +124,7 @@ export class PackageController {
|
|
|
100
124
|
bodyLocation,
|
|
101
125
|
stagingPath,
|
|
102
126
|
),
|
|
103
|
-
(pkg) => pkg
|
|
127
|
+
(pkg) => formatPublishRejections(pkg),
|
|
104
128
|
);
|
|
105
129
|
} else {
|
|
106
130
|
result = await environment.addPackage(packageName);
|
|
@@ -114,7 +138,7 @@ export class PackageController {
|
|
|
114
138
|
}
|
|
115
139
|
|
|
116
140
|
if (!body.location) {
|
|
117
|
-
const invalidMsg = result
|
|
141
|
+
const invalidMsg = formatPublishRejections(result);
|
|
118
142
|
if (invalidMsg) {
|
|
119
143
|
await environment.unloadPackage(packageName).catch(() => {
|
|
120
144
|
/* best-effort; the package is not persisted below */
|
|
@@ -177,7 +201,8 @@ export class PackageController {
|
|
|
177
201
|
stagingPath,
|
|
178
202
|
),
|
|
179
203
|
(pkg) =>
|
|
180
|
-
|
|
204
|
+
formatPublishRejections(
|
|
205
|
+
pkg,
|
|
181
206
|
body.explores?.map(normalizeModelPath),
|
|
182
207
|
),
|
|
183
208
|
);
|
|
@@ -81,6 +81,7 @@ import { fileURLToPath, pathToFileURL } from "url";
|
|
|
81
81
|
|
|
82
82
|
import { ModelCompilationError } from "../errors";
|
|
83
83
|
import { logger } from "../logger";
|
|
84
|
+
import type { PackageMaterializationConfig } from "../service/package_manifest";
|
|
84
85
|
import type {
|
|
85
86
|
ConnectionMetadataRequest,
|
|
86
87
|
ConnectionMetadataResponse,
|
|
@@ -233,7 +234,7 @@ export interface LoadPackageOutcome {
|
|
|
233
234
|
explores?: string[];
|
|
234
235
|
queryableSources?: "declared" | "all";
|
|
235
236
|
manifestLocation?: string | null;
|
|
236
|
-
materialization?:
|
|
237
|
+
materialization?: PackageMaterializationConfig | null;
|
|
237
238
|
};
|
|
238
239
|
models: Array<
|
|
239
240
|
Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
|
|
@@ -83,7 +83,10 @@ import { HackyDataStylesAccumulator } from "../data_styles";
|
|
|
83
83
|
import { ModelCompilationError } from "../errors";
|
|
84
84
|
import { validateAuthorizeProbes } from "../service/authorize";
|
|
85
85
|
import { type FilterDefinition } from "../service/filter";
|
|
86
|
-
import {
|
|
86
|
+
import {
|
|
87
|
+
PackageMaterializationConfig,
|
|
88
|
+
parsePackageMaterialization,
|
|
89
|
+
} from "../service/package_manifest";
|
|
87
90
|
import {
|
|
88
91
|
extractQueriesFromModelDef,
|
|
89
92
|
extractSourcesFromModelDef,
|
|
@@ -383,7 +386,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
|
|
|
383
386
|
explores?: string[];
|
|
384
387
|
queryableSources?: "declared" | "all";
|
|
385
388
|
manifestLocation?: string | null;
|
|
386
|
-
materialization?:
|
|
389
|
+
materialization?: PackageMaterializationConfig | null;
|
|
387
390
|
}> {
|
|
388
391
|
const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
|
|
389
392
|
const contents = await fs.promises.readFile(manifestPath, "utf8");
|
|
@@ -65,6 +65,7 @@
|
|
|
65
65
|
*/
|
|
66
66
|
|
|
67
67
|
import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
|
|
68
|
+
import type { PackageMaterializationConfig } from "../service/package_manifest";
|
|
68
69
|
|
|
69
70
|
// ──────────────────────────────────────────────────────────────────────
|
|
70
71
|
// Direction: main ──▶ worker (load-package job)
|
|
@@ -179,7 +180,7 @@ export interface LoadPackageResult {
|
|
|
179
180
|
explores?: string[];
|
|
180
181
|
queryableSources?: "declared" | "all";
|
|
181
182
|
manifestLocation?: string | null;
|
|
182
|
-
materialization?:
|
|
183
|
+
materialization?: PackageMaterializationConfig | null;
|
|
183
184
|
};
|
|
184
185
|
models: SerializedModel[];
|
|
185
186
|
/** Wall-clock ms inside the worker for the full package load. */
|
|
@@ -212,6 +212,50 @@ describe("deriveBuildPlan", () => {
|
|
|
212
212
|
});
|
|
213
213
|
});
|
|
214
214
|
|
|
215
|
+
it("reports declared sharing/refresh verbatim and null when unset", () => {
|
|
216
|
+
// The control plane distinguishes unset from an explicit `shared` (it
|
|
217
|
+
// applies the platform default itself), so the publisher must report
|
|
218
|
+
// the declared value verbatim — never substitute the default.
|
|
219
|
+
const declared = fakeSource({
|
|
220
|
+
name: "declared",
|
|
221
|
+
sourceEntityId: "bid-d",
|
|
222
|
+
annotationFields: {
|
|
223
|
+
name: "d_table",
|
|
224
|
+
sharing: "private",
|
|
225
|
+
refresh: "incremental",
|
|
226
|
+
},
|
|
227
|
+
});
|
|
228
|
+
const unset = fakeSource({ name: "unset", sourceEntityId: "bid-u" });
|
|
229
|
+
const plan = deriveBuildPlan(
|
|
230
|
+
[
|
|
231
|
+
{
|
|
232
|
+
connectionName: "duckdb",
|
|
233
|
+
nodes: [
|
|
234
|
+
[
|
|
235
|
+
{ sourceID: "declared@m", dependsOn: [] },
|
|
236
|
+
{ sourceID: "unset@m", dependsOn: [] },
|
|
237
|
+
],
|
|
238
|
+
],
|
|
239
|
+
},
|
|
240
|
+
] as unknown as Parameters<typeof deriveBuildPlan>[0],
|
|
241
|
+
{ "declared@m": declared, "unset@m": unset },
|
|
242
|
+
{ duckdb: "dig" },
|
|
243
|
+
);
|
|
244
|
+
|
|
245
|
+
expect(plan.sources["declared@m"].sharing).toBe("private");
|
|
246
|
+
expect(plan.sources["declared@m"].refresh).toBe("incremental");
|
|
247
|
+
// The raw annotation map still carries every field alongside the typed
|
|
248
|
+
// projections.
|
|
249
|
+
expect(plan.sources["declared@m"].annotationFields).toEqual({
|
|
250
|
+
name: "d_table",
|
|
251
|
+
sharing: "private",
|
|
252
|
+
refresh: "incremental",
|
|
253
|
+
});
|
|
254
|
+
// Unset is null — not "shared" — on the wire.
|
|
255
|
+
expect(plan.sources["unset@m"].sharing).toBeNull();
|
|
256
|
+
expect(plan.sources["unset@m"].refresh).toBeNull();
|
|
257
|
+
});
|
|
258
|
+
|
|
215
259
|
it("honors the sourceNames filter", () => {
|
|
216
260
|
const a = fakeSource({ name: "a", sourceEntityId: "bid-a" });
|
|
217
261
|
const b = fakeSource({ name: "b", sourceEntityId: "bid-b" });
|
|
@@ -307,6 +307,7 @@ export function deriveBuildPlan(
|
|
|
307
307
|
const wireSources: Record<string, WirePersistSourcePlan> = {};
|
|
308
308
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
309
309
|
if (include && !include.has(source.name)) continue;
|
|
310
|
+
const annotationFields = deriveAnnotationFields(source);
|
|
310
311
|
wireSources[sourceID] = {
|
|
311
312
|
name: source.name,
|
|
312
313
|
sourceID: source.sourceID,
|
|
@@ -314,8 +315,16 @@ export function deriveBuildPlan(
|
|
|
314
315
|
dialect: source.dialectName,
|
|
315
316
|
sourceEntityId: computeSourceEntityId(source, connectionDigests),
|
|
316
317
|
sql: source.getSQL(),
|
|
318
|
+
// Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
|
|
319
|
+
// = unset). The control plane applies the platform default (unset =>
|
|
320
|
+
// shared) itself, so the publisher must never substitute it — unset
|
|
321
|
+
// has to stay distinguishable from an explicit `shared`. The source's
|
|
322
|
+
// own annotation is the most-specific declaration layer, and the only
|
|
323
|
+
// one that exists today.
|
|
324
|
+
sharing: annotationFields.sharing ?? null,
|
|
325
|
+
refresh: annotationFields.refresh ?? null,
|
|
317
326
|
columns: deriveColumns(source),
|
|
318
|
-
annotationFields
|
|
327
|
+
annotationFields,
|
|
319
328
|
modelPath: sourceModelPaths?.[sourceID],
|
|
320
329
|
};
|
|
321
330
|
}
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it } from "bun:test";
|
|
2
|
+
import * as fs from "fs/promises";
|
|
3
|
+
import * as os from "os";
|
|
4
|
+
import * as path from "path";
|
|
5
|
+
|
|
6
|
+
import { Environment } from "./environment";
|
|
7
|
+
import type { Package } from "./package";
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* The publish gate for the package-level materialization cron (§9.4
|
|
11
|
+
* artifact-anchored scheduling): a declared `materialization.schedule` is the
|
|
12
|
+
* power tier and is valid only when every persist source in the compiled
|
|
13
|
+
* build plan resolves to an explicit `sharing=private`. These tests run a real
|
|
14
|
+
* `Environment` + `Package.create` over temp dirs, so they also prove
|
|
15
|
+
* end-to-end that the pinned compiler ACCEPTS `sharing=` / `refresh=` keys in
|
|
16
|
+
* the `#@ persist` annotation and that the values survive to the wire build
|
|
17
|
+
* plan verbatim (unset stays null — never defaulted to "shared").
|
|
18
|
+
*/
|
|
19
|
+
describe("materialization cron gate", () => {
|
|
20
|
+
let rootDir: string;
|
|
21
|
+
let envPath: string;
|
|
22
|
+
|
|
23
|
+
async function loadPackage(model: string, schedule?: string) {
|
|
24
|
+
const dir = path.join(envPath, "pkg");
|
|
25
|
+
await fs.mkdir(dir, { recursive: true });
|
|
26
|
+
await fs.writeFile(
|
|
27
|
+
path.join(dir, "publisher.json"),
|
|
28
|
+
JSON.stringify({
|
|
29
|
+
name: "pkg",
|
|
30
|
+
description: "fixture",
|
|
31
|
+
...(schedule ? { materialization: { schedule } } : {}),
|
|
32
|
+
}),
|
|
33
|
+
);
|
|
34
|
+
await fs.writeFile(path.join(dir, "model.malloy"), model);
|
|
35
|
+
const env = await Environment.create("testEnv", envPath, []);
|
|
36
|
+
await env.addPackage("pkg");
|
|
37
|
+
return env.getPackage("pkg", false);
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function sourceByName(pkg: Package, name: string) {
|
|
41
|
+
const sources = pkg.getBuildPlan()?.sources ?? {};
|
|
42
|
+
const found = Object.values(sources).find((s) => s.name === name);
|
|
43
|
+
if (!found) throw new Error(`persist source '${name}' not in build plan`);
|
|
44
|
+
return found;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
beforeEach(async () => {
|
|
48
|
+
rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-cron-"));
|
|
49
|
+
envPath = path.join(rootDir, "env");
|
|
50
|
+
await fs.mkdir(envPath, { recursive: true });
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
afterEach(async () => {
|
|
54
|
+
await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
const MIXED_MODEL = `##! experimental.persistence
|
|
58
|
+
|
|
59
|
+
#@ persist name="priv_table" sharing=private refresh=incremental
|
|
60
|
+
source: priv is duckdb.sql("SELECT 1 as x")
|
|
61
|
+
|
|
62
|
+
#@ persist name="open_table" sharing=shared
|
|
63
|
+
source: open is duckdb.sql("SELECT 2 as x")
|
|
64
|
+
|
|
65
|
+
#@ persist name="unset_table"
|
|
66
|
+
source: unspecified is duckdb.sql("SELECT 3 as x")
|
|
67
|
+
`;
|
|
68
|
+
|
|
69
|
+
const ALL_PRIVATE_MODEL = `##! experimental.persistence
|
|
70
|
+
|
|
71
|
+
#@ persist name="a_table" sharing=private
|
|
72
|
+
source: a is duckdb.sql("SELECT 1 as x")
|
|
73
|
+
|
|
74
|
+
#@ persist name="b_table" sharing=private refresh=full
|
|
75
|
+
source: b is duckdb.sql("SELECT 2 as x")
|
|
76
|
+
`;
|
|
77
|
+
|
|
78
|
+
it(
|
|
79
|
+
"surfaces declared sharing/refresh verbatim on the build plan (null when unset)",
|
|
80
|
+
async () => {
|
|
81
|
+
const pkg = await loadPackage(MIXED_MODEL);
|
|
82
|
+
|
|
83
|
+
const priv = sourceByName(pkg, "priv");
|
|
84
|
+
expect(priv.sharing).toBe("private");
|
|
85
|
+
expect(priv.refresh).toBe("incremental");
|
|
86
|
+
|
|
87
|
+
const open = sourceByName(pkg, "open");
|
|
88
|
+
expect(open.sharing).toBe("shared");
|
|
89
|
+
expect(open.refresh).toBeNull();
|
|
90
|
+
|
|
91
|
+
// Unset must be reported as null — distinguishable from an explicit
|
|
92
|
+
// "shared" — because the control plane applies the platform default
|
|
93
|
+
// (unset => shared) itself.
|
|
94
|
+
const unspecified = sourceByName(pkg, "unspecified");
|
|
95
|
+
expect(unspecified.sharing).toBeNull();
|
|
96
|
+
expect(unspecified.refresh).toBeNull();
|
|
97
|
+
},
|
|
98
|
+
{ timeout: 30000 },
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
it(
|
|
102
|
+
"rejects a cron over shared/unset persist sources, pointing at freshness.window",
|
|
103
|
+
async () => {
|
|
104
|
+
const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
|
|
105
|
+
|
|
106
|
+
const warnings = pkg.scheduleWarnings();
|
|
107
|
+
// One actionable message per offending source; the compliant private
|
|
108
|
+
// source is not flagged.
|
|
109
|
+
expect(warnings).toHaveLength(2);
|
|
110
|
+
const joined = pkg.formatInvalidSchedule();
|
|
111
|
+
expect(joined).toContain("'open' resolves to 'shared'");
|
|
112
|
+
expect(joined).toContain("'unspecified' resolves to unset");
|
|
113
|
+
expect(joined).not.toContain("'priv'");
|
|
114
|
+
expect(joined).toContain("materialization.freshness.window");
|
|
115
|
+
},
|
|
116
|
+
{ timeout: 30000 },
|
|
117
|
+
);
|
|
118
|
+
|
|
119
|
+
it(
|
|
120
|
+
"allows a cron when every persist source is explicitly private",
|
|
121
|
+
async () => {
|
|
122
|
+
const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
|
|
123
|
+
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
124
|
+
expect(pkg.formatInvalidSchedule()).toBe("");
|
|
125
|
+
},
|
|
126
|
+
{ timeout: 30000 },
|
|
127
|
+
);
|
|
128
|
+
|
|
129
|
+
it(
|
|
130
|
+
"is inert when no cron is declared, whatever the sources' sharing",
|
|
131
|
+
async () => {
|
|
132
|
+
const pkg = await loadPackage(MIXED_MODEL);
|
|
133
|
+
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
134
|
+
},
|
|
135
|
+
{ timeout: 30000 },
|
|
136
|
+
);
|
|
137
|
+
});
|