@malloy-publisher/server 0.0.224 → 0.0.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +58 -0
- package/dist/server.mjs +148 -16
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +1 -1
- package/src/controller/materialization.controller.spec.ts +72 -0
- package/src/controller/materialization.controller.ts +75 -11
- package/src/service/build_plan.spec.ts +119 -0
- package/src/service/build_plan.ts +143 -0
- package/src/service/materialization_cron_gate.spec.ts +60 -21
- package/src/service/materialization_service.spec.ts +42 -0
- package/src/service/materialization_service.ts +40 -2
- package/src/service/materialization_test_fixtures.ts +47 -14
- package/src/service/package.ts +64 -19
- package/src/storage/DatabaseInterface.ts +1 -0
- package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
- package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
- package/tests/fixtures/persist-multi-level/publisher.json +5 -0
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
deriveBuildPlan,
|
|
11
11
|
flattenDependsOn,
|
|
12
12
|
iterGraphSources,
|
|
13
|
+
resolveFreshnessSchedule,
|
|
13
14
|
resolvePackageConnections,
|
|
14
15
|
} from "./build_plan";
|
|
15
16
|
import { fakeSource } from "./materialization_test_fixtures";
|
|
@@ -296,6 +297,124 @@ describe("deriveBuildPlan", () => {
|
|
|
296
297
|
});
|
|
297
298
|
});
|
|
298
299
|
|
|
300
|
+
describe("resolveFreshnessSchedule", () => {
|
|
301
|
+
it("reports source-level freshness + schedule verbatim", () => {
|
|
302
|
+
const source = fakeSource({
|
|
303
|
+
name: "s",
|
|
304
|
+
sourceEntityId: "bid",
|
|
305
|
+
freshnessSchedule: {
|
|
306
|
+
freshness: { window: "1h", fallback: "stale_ok" },
|
|
307
|
+
schedule: "0 */6 * * *",
|
|
308
|
+
},
|
|
309
|
+
});
|
|
310
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
311
|
+
freshness: { window: "1h", fallback: "stale_ok" },
|
|
312
|
+
schedule: "0 */6 * * *",
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
it("returns null for both when unset at every level", () => {
|
|
317
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
318
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
319
|
+
freshness: null,
|
|
320
|
+
schedule: null,
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
it("falls back to model-file then package per field (most-specific-wins)", () => {
|
|
325
|
+
// freshness.window from source, freshness.fallback from model-file. The
|
|
326
|
+
// package-level cron is NOT inherited as the source's effective schedule;
|
|
327
|
+
// schedule here comes only from the source's own model-file default.
|
|
328
|
+
const source = fakeSource({
|
|
329
|
+
name: "s",
|
|
330
|
+
sourceEntityId: "bid",
|
|
331
|
+
freshnessSchedule: { freshness: { window: "1h" } },
|
|
332
|
+
modelFreshnessSchedule: {
|
|
333
|
+
freshness: { fallback: "fail" },
|
|
334
|
+
schedule: "0 3 * * *",
|
|
335
|
+
},
|
|
336
|
+
});
|
|
337
|
+
const pkg = {
|
|
338
|
+
schedule: "0 0 * * *",
|
|
339
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
340
|
+
};
|
|
341
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
342
|
+
freshness: { window: "1h", fallback: "fail" },
|
|
343
|
+
schedule: "0 3 * * *",
|
|
344
|
+
});
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it("does not inherit the package-level cron as the source's schedule", () => {
|
|
348
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
349
|
+
const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
|
|
350
|
+
// Freshness inherits from the package; schedule stays null (the package
|
|
351
|
+
// cron is gated separately, not folded into the per-source cron).
|
|
352
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
353
|
+
freshness: { window: "24h" },
|
|
354
|
+
schedule: null,
|
|
355
|
+
});
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it("inherits the package freshness when the source and model are unset", () => {
|
|
359
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
360
|
+
const pkg = { schedule: null, freshness: { window: "24h" } };
|
|
361
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
362
|
+
freshness: { window: "24h" },
|
|
363
|
+
schedule: null,
|
|
364
|
+
});
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it("drops an invalid fallback rather than defaulting it", () => {
|
|
368
|
+
const source = fakeSource({
|
|
369
|
+
name: "s",
|
|
370
|
+
sourceEntityId: "bid",
|
|
371
|
+
freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
|
|
372
|
+
});
|
|
373
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
374
|
+
freshness: { window: "1h" },
|
|
375
|
+
schedule: null,
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
describe("deriveBuildPlan freshness/schedule", () => {
|
|
381
|
+
it("projects the resolved per-source freshness + schedule onto the plan", () => {
|
|
382
|
+
const source = fakeSource({
|
|
383
|
+
name: "s",
|
|
384
|
+
sourceEntityId: "bid",
|
|
385
|
+
annotationFields: { name: "s_table", sharing: "private" },
|
|
386
|
+
freshnessSchedule: {
|
|
387
|
+
freshness: { window: "1h" },
|
|
388
|
+
schedule: "0 */6 * * *",
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
|
+
const plan = deriveBuildPlan(
|
|
392
|
+
[
|
|
393
|
+
{
|
|
394
|
+
connectionName: "duckdb",
|
|
395
|
+
nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
|
|
396
|
+
},
|
|
397
|
+
] as unknown as Parameters<typeof deriveBuildPlan>[0],
|
|
398
|
+
{ "s@m": source },
|
|
399
|
+
{ duckdb: "dig" },
|
|
400
|
+
undefined,
|
|
401
|
+
undefined,
|
|
402
|
+
{
|
|
403
|
+
schedule: null,
|
|
404
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
405
|
+
},
|
|
406
|
+
);
|
|
407
|
+
|
|
408
|
+
// Source window wins over the package default; package fallback fills the
|
|
409
|
+
// unset source fallback; source schedule reported verbatim.
|
|
410
|
+
expect(plan.sources["s@m"].freshness).toEqual({
|
|
411
|
+
window: "1h",
|
|
412
|
+
fallback: "live",
|
|
413
|
+
});
|
|
414
|
+
expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
|
|
415
|
+
});
|
|
416
|
+
});
|
|
417
|
+
|
|
299
418
|
describe("compilePackageBuildPlan", () => {
|
|
300
419
|
it("skips .malloynb notebooks without compiling them", async () => {
|
|
301
420
|
// A notebook would throw on its `>>>` cell delimiter if compiled as a
|
|
@@ -17,6 +17,25 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
|
|
|
17
17
|
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
18
18
|
type WireColumn = components["schemas"]["Column"];
|
|
19
19
|
type BuildPlan = components["schemas"]["BuildPlan"];
|
|
20
|
+
type WireFreshness = components["schemas"]["Freshness"];
|
|
21
|
+
type WirePackageMaterialization =
|
|
22
|
+
components["schemas"]["PackageMaterializationConfig"];
|
|
23
|
+
|
|
24
|
+
/** The freshness `fallback` values the publisher recognizes; others are dropped. */
|
|
25
|
+
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
26
|
+
type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
27
|
+
|
|
28
|
+
/** One layer's contribution to the resolved freshness/schedule (all optional). */
|
|
29
|
+
interface FreshnessScheduleLayer {
|
|
30
|
+
window?: string;
|
|
31
|
+
fallback?: FreshnessFallback;
|
|
32
|
+
schedule?: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
|
|
36
|
+
interface ReadableTag {
|
|
37
|
+
text(...path: string[]): string | undefined;
|
|
38
|
+
}
|
|
20
39
|
|
|
21
40
|
/**
|
|
22
41
|
* Minimal surface a package must expose to compile its build plan. Both
|
|
@@ -28,6 +47,13 @@ export interface BuildPlanPackage {
|
|
|
28
47
|
getPackagePath(): string;
|
|
29
48
|
getMalloyConfig(): MalloyConfig;
|
|
30
49
|
getMalloyConnection(name: string): Promise<MalloyConnection>;
|
|
50
|
+
/**
|
|
51
|
+
* The package-level `materialization` config (from malloy-publisher.json),
|
|
52
|
+
* used as the least-specific layer when resolving per-source freshness /
|
|
53
|
+
* schedule. Optional so existing fixtures/callers that don't track it still
|
|
54
|
+
* typecheck (they resolve without a package default).
|
|
55
|
+
*/
|
|
56
|
+
getMaterializationConfig?(): WirePackageMaterialization | null;
|
|
31
57
|
}
|
|
32
58
|
|
|
33
59
|
/**
|
|
@@ -94,6 +120,110 @@ export function deriveAnnotationFields(
|
|
|
94
120
|
return out;
|
|
95
121
|
}
|
|
96
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
|
|
125
|
+
* `schedule`) from one Malloy tag layer, keeping only recognized values. These
|
|
126
|
+
* are dotted/nested tag properties, so they are NOT captured by the scalar
|
|
127
|
+
* {@link deriveAnnotationFields} loop and must be read by path here.
|
|
128
|
+
*/
|
|
129
|
+
function tagFreshnessScheduleLayer(
|
|
130
|
+
tag: ReadableTag | undefined,
|
|
131
|
+
): FreshnessScheduleLayer {
|
|
132
|
+
if (!tag || typeof tag.text !== "function") return {};
|
|
133
|
+
const layer: FreshnessScheduleLayer = {};
|
|
134
|
+
const window = tag.text("freshness", "window");
|
|
135
|
+
if (typeof window === "string") layer.window = window;
|
|
136
|
+
const fallback = tag.text("freshness", "fallback");
|
|
137
|
+
if (
|
|
138
|
+
typeof fallback === "string" &&
|
|
139
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
140
|
+
) {
|
|
141
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
142
|
+
}
|
|
143
|
+
const schedule = tag.text("schedule");
|
|
144
|
+
if (typeof schedule === "string") layer.schedule = schedule;
|
|
145
|
+
return layer;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The package-level `materialization` config as a freshness resolution layer.
|
|
150
|
+
* Only `freshness` inherits down to a source: the package-level cron
|
|
151
|
+
* (`materialization.schedule`) is a distinct package-grain concept surfaced on
|
|
152
|
+
* `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
|
|
153
|
+
* folded into a source's effective per-source cron (doing so would double-count
|
|
154
|
+
* against the package cron gate).
|
|
155
|
+
*/
|
|
156
|
+
function packageFreshnessLayer(
|
|
157
|
+
cfg: WirePackageMaterialization | null | undefined,
|
|
158
|
+
): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
|
|
159
|
+
if (!cfg) return {};
|
|
160
|
+
const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
|
|
161
|
+
if (cfg.freshness?.window) layer.window = cfg.freshness.window;
|
|
162
|
+
const fallback = cfg.freshness?.fallback;
|
|
163
|
+
if (
|
|
164
|
+
typeof fallback === "string" &&
|
|
165
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
166
|
+
) {
|
|
167
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
168
|
+
}
|
|
169
|
+
return layer;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
|
|
173
|
+
function safeSourceTag(source: PersistSource): ReadableTag | undefined {
|
|
174
|
+
try {
|
|
175
|
+
return source.annotations.parseAsTag("@").tag as ReadableTag;
|
|
176
|
+
} catch {
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The model-file-level (`##`) tag resolved for the source, or undefined on a
|
|
183
|
+
* parse failure. The model-file layer sits between the source annotation and
|
|
184
|
+
* the package default in most-specific-wins resolution.
|
|
185
|
+
*/
|
|
186
|
+
function safeModelTag(source: PersistSource): ReadableTag | undefined {
|
|
187
|
+
try {
|
|
188
|
+
return source.modelAnnotations.parseAsTag().tag as ReadableTag;
|
|
189
|
+
} catch {
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
|
|
196
|
+
* at every level stays null so the control plane can distinguish "unset" —
|
|
197
|
+
* apply platform default — from an explicit declaration). Invalid `fallback`
|
|
198
|
+
* values are dropped, never defaulted. Precedence, per field, most-specific-wins:
|
|
199
|
+
*
|
|
200
|
+
* - `freshness`: source annotation > model-file default > package default.
|
|
201
|
+
* - `schedule`: source annotation > model-file default (the source's OWN cron).
|
|
202
|
+
* The package-level cron is a package-grain concept, surfaced and gated
|
|
203
|
+
* separately, so it is not folded into the per-source effective cron.
|
|
204
|
+
*/
|
|
205
|
+
export function resolveFreshnessSchedule(
|
|
206
|
+
source: PersistSource,
|
|
207
|
+
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
208
|
+
): { freshness: WireFreshness | null; schedule: string | null } {
|
|
209
|
+
const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
|
|
210
|
+
const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
|
|
211
|
+
const pkgLayer = packageFreshnessLayer(packageMaterialization);
|
|
212
|
+
|
|
213
|
+
const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
|
|
214
|
+
const fallback =
|
|
215
|
+
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
216
|
+
const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
|
|
217
|
+
|
|
218
|
+
let freshness: WireFreshness | null = null;
|
|
219
|
+
if (window !== undefined || fallback !== undefined) {
|
|
220
|
+
freshness = {};
|
|
221
|
+
if (window !== undefined) freshness.window = window;
|
|
222
|
+
if (fallback !== undefined) freshness.fallback = fallback;
|
|
223
|
+
}
|
|
224
|
+
return { freshness, schedule };
|
|
225
|
+
}
|
|
226
|
+
|
|
97
227
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
98
228
|
export function flattenDependsOn(node: {
|
|
99
229
|
dependsOn: { sourceID: string }[];
|
|
@@ -291,6 +421,7 @@ export function deriveBuildPlan(
|
|
|
291
421
|
connectionDigests: Record<string, string>,
|
|
292
422
|
sourceNames?: string[],
|
|
293
423
|
sourceModelPaths?: Record<string, string>,
|
|
424
|
+
packageMaterialization?: WirePackageMaterialization | null,
|
|
294
425
|
): BuildPlan {
|
|
295
426
|
const include = sourceNames ? new Set(sourceNames) : null;
|
|
296
427
|
|
|
@@ -308,6 +439,15 @@ export function deriveBuildPlan(
|
|
|
308
439
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
309
440
|
if (include && !include.has(source.name)) continue;
|
|
310
441
|
const annotationFields = deriveAnnotationFields(source);
|
|
442
|
+
// EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
|
|
443
|
+
// (source > model-file > package) and reported verbatim (null = unset at
|
|
444
|
+
// every level). Unlike `sharing`/`refresh` these are dotted/nested tag
|
|
445
|
+
// keys, so they come from resolveFreshnessSchedule rather than the scalar
|
|
446
|
+
// annotationFields map.
|
|
447
|
+
const { freshness, schedule } = resolveFreshnessSchedule(
|
|
448
|
+
source,
|
|
449
|
+
packageMaterialization,
|
|
450
|
+
);
|
|
311
451
|
wireSources[sourceID] = {
|
|
312
452
|
name: source.name,
|
|
313
453
|
sourceID: source.sourceID,
|
|
@@ -323,6 +463,8 @@ export function deriveBuildPlan(
|
|
|
323
463
|
// one that exists today.
|
|
324
464
|
sharing: annotationFields.sharing ?? null,
|
|
325
465
|
refresh: annotationFields.refresh ?? null,
|
|
466
|
+
freshness,
|
|
467
|
+
schedule,
|
|
326
468
|
columns: deriveColumns(source),
|
|
327
469
|
annotationFields,
|
|
328
470
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -351,5 +493,6 @@ export async function computePackageBuildPlan(
|
|
|
351
493
|
compiled.connectionDigests,
|
|
352
494
|
undefined,
|
|
353
495
|
compiled.sourceModelPaths,
|
|
496
|
+
pkg.getMaterializationConfig?.() ?? null,
|
|
354
497
|
);
|
|
355
498
|
}
|
|
@@ -7,17 +7,22 @@ import { Environment } from "./environment";
|
|
|
7
7
|
import type { Package } from "./package";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
* The publish gate for
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
10
|
+
* The publish gate for materialization crons (Phase A carve-out,
|
|
11
|
+
* persistence.md §9.4). A cron is the power tier of artifact-anchored
|
|
12
|
+
* scheduling and is valid only over `sharing="private"` artifacts:
|
|
13
|
+
*
|
|
14
|
+
* - a package-level cron (`materialization.schedule`) is valid only when
|
|
15
|
+
* EVERY governed persist source resolves to explicit `sharing="private"`;
|
|
16
|
+
* - a per-source cron (`#@ persist ... schedule=`) is valid only when THAT
|
|
17
|
+
* source resolves to explicit `sharing="private"`;
|
|
18
|
+
* - a cron on a shared/unset artifact is rejected, pointing at
|
|
19
|
+
* `freshness.window` as the shared-tier replacement.
|
|
20
|
+
*
|
|
21
|
+
* These tests run a real `Environment` + `Package.create` over temp dirs, so
|
|
22
|
+
* they also prove end-to-end that the pinned compiler ACCEPTS `sharing=` /
|
|
23
|
+
* `refresh=` / `schedule=` / `freshness.*` keys in the `#@ persist` annotation
|
|
24
|
+
* and that the values survive to the wire build plan verbatim (unset stays null
|
|
25
|
+
* — never defaulted to "shared").
|
|
21
26
|
*/
|
|
22
27
|
describe("materialization cron gate", () => {
|
|
23
28
|
let rootDir: string;
|
|
@@ -76,6 +81,20 @@ source: a is duckdb.sql("SELECT 1 as x")
|
|
|
76
81
|
|
|
77
82
|
#@ persist name="b_table" sharing=private refresh=full
|
|
78
83
|
source: b is duckdb.sql("SELECT 2 as x")
|
|
84
|
+
`;
|
|
85
|
+
|
|
86
|
+
// Per-source crons: valid on the private source, rejected on the shared and
|
|
87
|
+
// the unset (⇒ shared) sources.
|
|
88
|
+
const PER_SOURCE_CRON_MODEL = `##! experimental.persistence
|
|
89
|
+
|
|
90
|
+
#@ persist name="p_table" sharing=private schedule="0 */6 * * *"
|
|
91
|
+
source: p is duckdb.sql("SELECT 1 as x")
|
|
92
|
+
|
|
93
|
+
#@ persist name="s_table" sharing=shared schedule="0 0 * * *"
|
|
94
|
+
source: s is duckdb.sql("SELECT 2 as x")
|
|
95
|
+
|
|
96
|
+
#@ persist name="u_table" schedule="0 0 * * *"
|
|
97
|
+
source: u is duckdb.sql("SELECT 3 as x")
|
|
79
98
|
`;
|
|
80
99
|
|
|
81
100
|
it(
|
|
@@ -102,12 +121,13 @@ source: b is duckdb.sql("SELECT 2 as x")
|
|
|
102
121
|
);
|
|
103
122
|
|
|
104
123
|
it(
|
|
105
|
-
"rejects
|
|
124
|
+
"rejects a package cron on a mixed package, pointing at freshness.window",
|
|
106
125
|
async () => {
|
|
126
|
+
// Phase A: a package cron governs every persist source, so it is valid
|
|
127
|
+
// only when all resolve to explicit private. MIXED has a shared and an
|
|
128
|
+
// unset source, so the package cron is rejected.
|
|
107
129
|
const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
|
|
108
130
|
|
|
109
|
-
// One actionable message for the whole package — the B→A rule is
|
|
110
|
-
// reject-all, not per-source.
|
|
111
131
|
const warnings = pkg.scheduleWarnings();
|
|
112
132
|
expect(warnings).toHaveLength(1);
|
|
113
133
|
const joined = pkg.formatInvalidSchedule();
|
|
@@ -118,15 +138,34 @@ source: b is duckdb.sql("SELECT 2 as x")
|
|
|
118
138
|
);
|
|
119
139
|
|
|
120
140
|
it(
|
|
121
|
-
"
|
|
141
|
+
"accepts a package cron when every persist source is private",
|
|
122
142
|
async () => {
|
|
123
|
-
//
|
|
124
|
-
//
|
|
143
|
+
// Phase A carve-out: an all-private package legitimately carries a
|
|
144
|
+
// package-level cron (the power tier for its own private artifacts).
|
|
125
145
|
const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
|
|
126
|
-
expect(pkg.scheduleWarnings()).
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
146
|
+
expect(pkg.scheduleWarnings()).toEqual([]);
|
|
147
|
+
},
|
|
148
|
+
{ timeout: 30000 },
|
|
149
|
+
);
|
|
150
|
+
|
|
151
|
+
it(
|
|
152
|
+
"rejects a per-source cron unless that source resolves to private",
|
|
153
|
+
async () => {
|
|
154
|
+
// No package cron here — only per-source `schedule=` declarations. The
|
|
155
|
+
// private source's cron is accepted; the shared and unset sources' are
|
|
156
|
+
// rejected, each pointing at freshness.window.
|
|
157
|
+
const pkg = await loadPackage(PER_SOURCE_CRON_MODEL);
|
|
158
|
+
|
|
159
|
+
const warnings = pkg.scheduleWarnings();
|
|
160
|
+
expect(warnings).toHaveLength(2);
|
|
161
|
+
const joined = warnings.join("\n");
|
|
162
|
+
expect(joined).toContain('"s"');
|
|
163
|
+
expect(joined).toContain('"u"');
|
|
164
|
+
expect(joined).not.toContain('"p"');
|
|
165
|
+
expect(joined).toContain("freshness.window");
|
|
166
|
+
|
|
167
|
+
// The resolved per-source schedule is surfaced verbatim on the plan.
|
|
168
|
+
expect(sourceByName(pkg, "p").schedule).toBe("0 */6 * * *");
|
|
130
169
|
},
|
|
131
170
|
{ timeout: 30000 },
|
|
132
171
|
);
|
|
@@ -921,6 +921,10 @@ describe("runBuild (branch behavior)", () => {
|
|
|
921
921
|
sourceNames: string[] | undefined;
|
|
922
922
|
forceRefresh: boolean;
|
|
923
923
|
buildInstructions: BuildInstruction[] | undefined;
|
|
924
|
+
referenceManifest?:
|
|
925
|
+
| { sourceEntityId: string; physicalTableName: string }[]
|
|
926
|
+
| undefined;
|
|
927
|
+
strictUpstreams?: boolean | undefined;
|
|
924
928
|
},
|
|
925
929
|
signal: AbortSignal,
|
|
926
930
|
) => Promise<void>;
|
|
@@ -972,6 +976,44 @@ describe("runBuild (branch behavior)", () => {
|
|
|
972
976
|
expect(svc.autoLoadManifest.called).toBe(false);
|
|
973
977
|
});
|
|
974
978
|
|
|
979
|
+
it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
|
|
980
|
+
const svc = stubEngine();
|
|
981
|
+
const instructions = [makeInstruction()];
|
|
982
|
+
|
|
983
|
+
await svc.runBuild(
|
|
984
|
+
"mat-1",
|
|
985
|
+
"my-env",
|
|
986
|
+
"pkg",
|
|
987
|
+
{
|
|
988
|
+
sourceNames: undefined,
|
|
989
|
+
forceRefresh: false,
|
|
990
|
+
buildInstructions: instructions,
|
|
991
|
+
referenceManifest: [
|
|
992
|
+
{ sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
|
|
993
|
+
],
|
|
994
|
+
strictUpstreams: true,
|
|
995
|
+
},
|
|
996
|
+
new AbortController().signal,
|
|
997
|
+
);
|
|
998
|
+
|
|
999
|
+
// The reference manifest becomes the seed (carried) entries so a
|
|
1000
|
+
// downstream build resolves its upstream to the existing physical table.
|
|
1001
|
+
expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
|
|
1002
|
+
"up-1": {
|
|
1003
|
+
sourceEntityId: "up-1",
|
|
1004
|
+
physicalTableName: "upstream_tbl",
|
|
1005
|
+
},
|
|
1006
|
+
});
|
|
1007
|
+
// strictUpstreams flows through as the strict flag (5th arg).
|
|
1008
|
+
expect(svc.executeInstructedBuild.firstCall.args[4]).toBe(true);
|
|
1009
|
+
// Reused upstreams are counted as carried, not built.
|
|
1010
|
+
expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
|
|
1011
|
+
mode: "orchestrated",
|
|
1012
|
+
sourcesBuilt: 1,
|
|
1013
|
+
sourcesReused: 1,
|
|
1014
|
+
});
|
|
1015
|
+
});
|
|
1016
|
+
|
|
975
1017
|
it("auto-run: derives instructions and auto-loads the manifest", async () => {
|
|
976
1018
|
const svc = stubEngine();
|
|
977
1019
|
|
|
@@ -28,6 +28,7 @@ import {
|
|
|
28
28
|
MaterializationStatus,
|
|
29
29
|
MaterializationUpdate,
|
|
30
30
|
ManifestEntry,
|
|
31
|
+
ManifestReference,
|
|
31
32
|
ResourceRepository,
|
|
32
33
|
} from "../storage/DatabaseInterface";
|
|
33
34
|
import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
|
|
@@ -216,6 +217,8 @@ export class MaterializationService {
|
|
|
216
217
|
forceRefresh?: boolean;
|
|
217
218
|
sourceNames?: string[];
|
|
218
219
|
buildInstructions?: BuildInstruction[];
|
|
220
|
+
referenceManifest?: ManifestReference[];
|
|
221
|
+
strictUpstreams?: boolean;
|
|
219
222
|
} = {},
|
|
220
223
|
): Promise<Materialization> {
|
|
221
224
|
const environmentId = await this.resolveEnvironmentId(environmentName);
|
|
@@ -275,6 +278,8 @@ export class MaterializationService {
|
|
|
275
278
|
sourceNames: options.sourceNames,
|
|
276
279
|
forceRefresh,
|
|
277
280
|
buildInstructions,
|
|
281
|
+
referenceManifest: options.referenceManifest,
|
|
282
|
+
strictUpstreams: options.strictUpstreams,
|
|
278
283
|
},
|
|
279
284
|
signal,
|
|
280
285
|
),
|
|
@@ -299,6 +304,8 @@ export class MaterializationService {
|
|
|
299
304
|
sourceNames: string[] | undefined;
|
|
300
305
|
forceRefresh: boolean;
|
|
301
306
|
buildInstructions: BuildInstruction[] | undefined;
|
|
307
|
+
referenceManifest: ManifestReference[] | undefined;
|
|
308
|
+
strictUpstreams: boolean | undefined;
|
|
302
309
|
},
|
|
303
310
|
signal: AbortSignal,
|
|
304
311
|
): Promise<void> {
|
|
@@ -333,7 +340,12 @@ export class MaterializationService {
|
|
|
333
340
|
let carried: Record<string, ManifestEntry>;
|
|
334
341
|
if (orchestrated) {
|
|
335
342
|
instructions = opts.buildInstructions!;
|
|
336
|
-
|
|
343
|
+
// Seed the build Manifest with the caller-supplied upstream
|
|
344
|
+
// references so a downstream source's persist upstream (built in a
|
|
345
|
+
// prior run, not rebuilt here) resolves to its existing physical
|
|
346
|
+
// table instead of recomputing live. The reference key is the
|
|
347
|
+
// compiler's manifest-lookup sourceEntityId (see ManifestReference).
|
|
348
|
+
carried = this.referenceManifestToEntries(opts.referenceManifest);
|
|
337
349
|
} else {
|
|
338
350
|
// Skip-if-unchanged: reuse tables from the most recent successful
|
|
339
351
|
// manifest for sources whose sourceEntityId is unchanged, unless
|
|
@@ -357,6 +369,7 @@ export class MaterializationService {
|
|
|
357
369
|
instructions,
|
|
358
370
|
carried,
|
|
359
371
|
signal,
|
|
372
|
+
opts.strictUpstreams ?? false,
|
|
360
373
|
);
|
|
361
374
|
|
|
362
375
|
const sourcesBuilt = instructions.length;
|
|
@@ -450,6 +463,26 @@ export class MaterializationService {
|
|
|
450
463
|
return { instructions, carried };
|
|
451
464
|
}
|
|
452
465
|
|
|
466
|
+
/**
|
|
467
|
+
* Project the caller-supplied upstream reference manifest into the seed
|
|
468
|
+
* entry map `executeInstructedBuild` consumes. Each reference is keyed by the
|
|
469
|
+
* compiler's manifest-lookup sourceEntityId and carries only the physical
|
|
470
|
+
* table name — enough for the build Manifest to resolve a downstream persist
|
|
471
|
+
* reference to the existing table without rebuilding the upstream.
|
|
472
|
+
*/
|
|
473
|
+
private referenceManifestToEntries(
|
|
474
|
+
referenceManifest: ManifestReference[] | undefined,
|
|
475
|
+
): Record<string, ManifestEntry> {
|
|
476
|
+
const entries: Record<string, ManifestEntry> = {};
|
|
477
|
+
for (const ref of referenceManifest ?? []) {
|
|
478
|
+
entries[ref.sourceEntityId] = {
|
|
479
|
+
sourceEntityId: ref.sourceEntityId,
|
|
480
|
+
physicalTableName: ref.physicalTableName,
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
return entries;
|
|
484
|
+
}
|
|
485
|
+
|
|
453
486
|
/**
|
|
454
487
|
* Entries of the most recent successful (MANIFEST_FILE_READY) materialization
|
|
455
488
|
* for this package, used for skip-if-unchanged. Excludes the in-flight run.
|
|
@@ -565,6 +598,7 @@ export class MaterializationService {
|
|
|
565
598
|
instructions: BuildInstruction[],
|
|
566
599
|
seedEntries: Record<string, ManifestEntry>,
|
|
567
600
|
signal: AbortSignal,
|
|
601
|
+
strict = false,
|
|
568
602
|
): Promise<Record<string, ManifestEntry>> {
|
|
569
603
|
const { graphs, sources, connectionDigests, connections } = compiled;
|
|
570
604
|
|
|
@@ -587,8 +621,12 @@ export class MaterializationService {
|
|
|
587
621
|
|
|
588
622
|
// Accumulates physical names as sources are built so downstream sources
|
|
589
623
|
// resolve their upstream references to the freshly-assigned tables. Seed
|
|
590
|
-
// it with carried-forward entries so reused upstreams resolve too.
|
|
624
|
+
// it with carried-forward entries so reused upstreams resolve too. In
|
|
625
|
+
// strict mode, an upstream persist reference that is neither built here
|
|
626
|
+
// nor seeded fails the compile (runtime-manifest-strict-miss) instead of
|
|
627
|
+
// silently recomputing live.
|
|
591
628
|
const manifest = new Manifest();
|
|
629
|
+
manifest.strict = strict;
|
|
592
630
|
const entries: Record<string, ManifestEntry> = {};
|
|
593
631
|
for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
|
|
594
632
|
if (entry.physicalTableName) {
|
|
@@ -75,6 +75,41 @@ export function makeInstruction(
|
|
|
75
75
|
* build internals touch (name/id, deterministic sourceEntityId, SQL, and the
|
|
76
76
|
* `#@ persist name=` annotation reader, defaulted to "unset").
|
|
77
77
|
*/
|
|
78
|
+
/** A freshness/schedule layer for the fake source's `#@` or `##` tag. */
|
|
79
|
+
interface FakeFreshnessSchedule {
|
|
80
|
+
freshness?: { window?: string; fallback?: string };
|
|
81
|
+
schedule?: string;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
/**
|
|
85
|
+
* Build a fake Malloy `Tag` supporting both readers the build plan uses:
|
|
86
|
+
* `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
|
|
87
|
+
* and the path-based `text(...at)` (dotted `freshness.window` / `schedule`, for
|
|
88
|
+
* resolveFreshnessSchedule).
|
|
89
|
+
*/
|
|
90
|
+
function fakeTag(
|
|
91
|
+
fields: Record<string, string> | undefined,
|
|
92
|
+
fs: FakeFreshnessSchedule | undefined,
|
|
93
|
+
) {
|
|
94
|
+
return {
|
|
95
|
+
*entries() {
|
|
96
|
+
for (const [key, value] of Object.entries(fields ?? {})) {
|
|
97
|
+
yield [key, { text: () => value }];
|
|
98
|
+
}
|
|
99
|
+
},
|
|
100
|
+
text(...at: string[]): string | undefined {
|
|
101
|
+
if (at.length === 1) {
|
|
102
|
+
if (at[0] === "schedule") return fs?.schedule;
|
|
103
|
+
return fields?.[at[0]];
|
|
104
|
+
}
|
|
105
|
+
if (at.length === 2 && at[0] === "freshness") {
|
|
106
|
+
return fs?.freshness?.[at[1] as "window" | "fallback"];
|
|
107
|
+
}
|
|
108
|
+
return undefined;
|
|
109
|
+
},
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
|
|
78
113
|
export function fakeSource(opts: {
|
|
79
114
|
name: string;
|
|
80
115
|
sourceEntityId: string;
|
|
@@ -83,6 +118,10 @@ export function fakeSource(opts: {
|
|
|
83
118
|
dialectName?: string;
|
|
84
119
|
/** key=value fields of the `#@ persist` annotation (e.g. sharing). */
|
|
85
120
|
annotationFields?: Record<string, string>;
|
|
121
|
+
/** Source-level (`#@`) freshness/schedule (dotted keys). */
|
|
122
|
+
freshnessSchedule?: FakeFreshnessSchedule;
|
|
123
|
+
/** Model-file-level (`##`) freshness/schedule default. */
|
|
124
|
+
modelFreshnessSchedule?: FakeFreshnessSchedule;
|
|
86
125
|
}): PersistSource {
|
|
87
126
|
const fields = opts.annotationFields;
|
|
88
127
|
return {
|
|
@@ -93,20 +132,14 @@ export function fakeSource(opts: {
|
|
|
93
132
|
makeBuildId: () => opts.sourceEntityId,
|
|
94
133
|
getSQL: () => opts.sql ?? "SELECT 1",
|
|
95
134
|
annotations: {
|
|
96
|
-
parseAsTag: () =>
|
|
97
|
-
fields
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
},
|
|
105
|
-
},
|
|
106
|
-
}
|
|
107
|
-
: // No annotation fields: entries() is absent, and
|
|
108
|
-
// deriveAnnotationFields degrades to {}.
|
|
109
|
-
{ tag: { text: () => undefined } },
|
|
135
|
+
parseAsTag: () => ({
|
|
136
|
+
tag: fakeTag(fields, opts.freshnessSchedule),
|
|
137
|
+
}),
|
|
138
|
+
},
|
|
139
|
+
modelAnnotations: {
|
|
140
|
+
parseAsTag: () => ({
|
|
141
|
+
tag: fakeTag(undefined, opts.modelFreshnessSchedule),
|
|
142
|
+
}),
|
|
110
143
|
},
|
|
111
144
|
} as unknown as PersistSource;
|
|
112
145
|
}
|