@malloy-publisher/server 0.0.224 → 0.0.225

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -10,6 +10,7 @@ import {
10
10
  deriveBuildPlan,
11
11
  flattenDependsOn,
12
12
  iterGraphSources,
13
+ resolveFreshnessSchedule,
13
14
  resolvePackageConnections,
14
15
  } from "./build_plan";
15
16
  import { fakeSource } from "./materialization_test_fixtures";
@@ -296,6 +297,124 @@ describe("deriveBuildPlan", () => {
296
297
  });
297
298
  });
298
299
 
300
+ describe("resolveFreshnessSchedule", () => {
301
+ it("reports source-level freshness + schedule verbatim", () => {
302
+ const source = fakeSource({
303
+ name: "s",
304
+ sourceEntityId: "bid",
305
+ freshnessSchedule: {
306
+ freshness: { window: "1h", fallback: "stale_ok" },
307
+ schedule: "0 */6 * * *",
308
+ },
309
+ });
310
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
311
+ freshness: { window: "1h", fallback: "stale_ok" },
312
+ schedule: "0 */6 * * *",
313
+ });
314
+ });
315
+
316
+ it("returns null for both when unset at every level", () => {
317
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
318
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
319
+ freshness: null,
320
+ schedule: null,
321
+ });
322
+ });
323
+
324
+ it("falls back to model-file then package per field (most-specific-wins)", () => {
325
+ // freshness.window from source, freshness.fallback from model-file. The
326
+ // package-level cron is NOT inherited as the source's effective schedule;
327
+ // schedule here comes only from the source's own model-file default.
328
+ const source = fakeSource({
329
+ name: "s",
330
+ sourceEntityId: "bid",
331
+ freshnessSchedule: { freshness: { window: "1h" } },
332
+ modelFreshnessSchedule: {
333
+ freshness: { fallback: "fail" },
334
+ schedule: "0 3 * * *",
335
+ },
336
+ });
337
+ const pkg = {
338
+ schedule: "0 0 * * *",
339
+ freshness: { window: "24h", fallback: "live" as const },
340
+ };
341
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
342
+ freshness: { window: "1h", fallback: "fail" },
343
+ schedule: "0 3 * * *",
344
+ });
345
+ });
346
+
347
+ it("does not inherit the package-level cron as the source's schedule", () => {
348
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
349
+ const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
350
+ // Freshness inherits from the package; schedule stays null (the package
351
+ // cron is gated separately, not folded into the per-source cron).
352
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
353
+ freshness: { window: "24h" },
354
+ schedule: null,
355
+ });
356
+ });
357
+
358
+ it("inherits the package freshness when the source and model are unset", () => {
359
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
360
+ const pkg = { schedule: null, freshness: { window: "24h" } };
361
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
362
+ freshness: { window: "24h" },
363
+ schedule: null,
364
+ });
365
+ });
366
+
367
+ it("drops an invalid fallback rather than defaulting it", () => {
368
+ const source = fakeSource({
369
+ name: "s",
370
+ sourceEntityId: "bid",
371
+ freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
372
+ });
373
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
374
+ freshness: { window: "1h" },
375
+ schedule: null,
376
+ });
377
+ });
378
+ });
379
+
380
+ describe("deriveBuildPlan freshness/schedule", () => {
381
+ it("projects the resolved per-source freshness + schedule onto the plan", () => {
382
+ const source = fakeSource({
383
+ name: "s",
384
+ sourceEntityId: "bid",
385
+ annotationFields: { name: "s_table", sharing: "private" },
386
+ freshnessSchedule: {
387
+ freshness: { window: "1h" },
388
+ schedule: "0 */6 * * *",
389
+ },
390
+ });
391
+ const plan = deriveBuildPlan(
392
+ [
393
+ {
394
+ connectionName: "duckdb",
395
+ nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
396
+ },
397
+ ] as unknown as Parameters<typeof deriveBuildPlan>[0],
398
+ { "s@m": source },
399
+ { duckdb: "dig" },
400
+ undefined,
401
+ undefined,
402
+ {
403
+ schedule: null,
404
+ freshness: { window: "24h", fallback: "live" as const },
405
+ },
406
+ );
407
+
408
+ // Source window wins over the package default; package fallback fills the
409
+ // unset source fallback; source schedule reported verbatim.
410
+ expect(plan.sources["s@m"].freshness).toEqual({
411
+ window: "1h",
412
+ fallback: "live",
413
+ });
414
+ expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
415
+ });
416
+ });
417
+
299
418
  describe("compilePackageBuildPlan", () => {
300
419
  it("skips .malloynb notebooks without compiling them", async () => {
301
420
  // A notebook would throw on its `>>>` cell delimiter if compiled as a
@@ -17,6 +17,25 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
17
17
  type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
18
18
  type WireColumn = components["schemas"]["Column"];
19
19
  type BuildPlan = components["schemas"]["BuildPlan"];
20
+ type WireFreshness = components["schemas"]["Freshness"];
21
+ type WirePackageMaterialization =
22
+ components["schemas"]["PackageMaterializationConfig"];
23
+
24
+ /** The freshness `fallback` values the publisher recognizes; others are dropped. */
25
+ const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
26
+ type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
27
+
28
+ /** One layer's contribution to the resolved freshness/schedule (all optional). */
29
+ interface FreshnessScheduleLayer {
30
+ window?: string;
31
+ fallback?: FreshnessFallback;
32
+ schedule?: string;
33
+ }
34
+
35
+ /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
36
+ interface ReadableTag {
37
+ text(...path: string[]): string | undefined;
38
+ }
20
39
 
21
40
  /**
22
41
  * Minimal surface a package must expose to compile its build plan. Both
@@ -28,6 +47,13 @@ export interface BuildPlanPackage {
28
47
  getPackagePath(): string;
29
48
  getMalloyConfig(): MalloyConfig;
30
49
  getMalloyConnection(name: string): Promise<MalloyConnection>;
50
+ /**
51
+ * The package-level `materialization` config (from malloy-publisher.json),
52
+ * used as the least-specific layer when resolving per-source freshness /
53
+ * schedule. Optional so existing fixtures/callers that don't track it still
54
+ * typecheck (they resolve without a package default).
55
+ */
56
+ getMaterializationConfig?(): WirePackageMaterialization | null;
31
57
  }
32
58
 
33
59
  /**
@@ -94,6 +120,110 @@ export function deriveAnnotationFields(
94
120
  return out;
95
121
  }
96
122
 
123
+ /**
124
+ * Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
125
+ * `schedule`) from one Malloy tag layer, keeping only recognized values. These
126
+ * are dotted/nested tag properties, so they are NOT captured by the scalar
127
+ * {@link deriveAnnotationFields} loop and must be read by path here.
128
+ */
129
+ function tagFreshnessScheduleLayer(
130
+ tag: ReadableTag | undefined,
131
+ ): FreshnessScheduleLayer {
132
+ if (!tag || typeof tag.text !== "function") return {};
133
+ const layer: FreshnessScheduleLayer = {};
134
+ const window = tag.text("freshness", "window");
135
+ if (typeof window === "string") layer.window = window;
136
+ const fallback = tag.text("freshness", "fallback");
137
+ if (
138
+ typeof fallback === "string" &&
139
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
140
+ ) {
141
+ layer.fallback = fallback as FreshnessFallback;
142
+ }
143
+ const schedule = tag.text("schedule");
144
+ if (typeof schedule === "string") layer.schedule = schedule;
145
+ return layer;
146
+ }
147
+
148
+ /**
149
+ * The package-level `materialization` config as a freshness resolution layer.
150
+ * Only `freshness` inherits down to a source: the package-level cron
151
+ * (`materialization.schedule`) is a distinct package-grain concept surfaced on
152
+ * `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
153
+ * folded into a source's effective per-source cron (doing so would double-count
154
+ * against the package cron gate).
155
+ */
156
+ function packageFreshnessLayer(
157
+ cfg: WirePackageMaterialization | null | undefined,
158
+ ): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
159
+ if (!cfg) return {};
160
+ const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
161
+ if (cfg.freshness?.window) layer.window = cfg.freshness.window;
162
+ const fallback = cfg.freshness?.fallback;
163
+ if (
164
+ typeof fallback === "string" &&
165
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
166
+ ) {
167
+ layer.fallback = fallback as FreshnessFallback;
168
+ }
169
+ return layer;
170
+ }
171
+
172
+ /** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
173
+ function safeSourceTag(source: PersistSource): ReadableTag | undefined {
174
+ try {
175
+ return source.annotations.parseAsTag("@").tag as ReadableTag;
176
+ } catch {
177
+ return undefined;
178
+ }
179
+ }
180
+
181
+ /**
182
+ * The model-file-level (`##`) tag resolved for the source, or undefined on a
183
+ * parse failure. The model-file layer sits between the source annotation and
184
+ * the package default in most-specific-wins resolution.
185
+ */
186
+ function safeModelTag(source: PersistSource): ReadableTag | undefined {
187
+ try {
188
+ return source.modelAnnotations.parseAsTag().tag as ReadableTag;
189
+ } catch {
190
+ return undefined;
191
+ }
192
+ }
193
+
194
+ /**
195
+ * Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
196
+ * at every level stays null so the control plane can distinguish "unset" —
197
+ * apply platform default — from an explicit declaration). Invalid `fallback`
198
+ * values are dropped, never defaulted. Precedence, per field, most-specific-wins:
199
+ *
200
+ * - `freshness`: source annotation > model-file default > package default.
201
+ * - `schedule`: source annotation > model-file default (the source's OWN cron).
202
+ * The package-level cron is a package-grain concept, surfaced and gated
203
+ * separately, so it is not folded into the per-source effective cron.
204
+ */
205
+ export function resolveFreshnessSchedule(
206
+ source: PersistSource,
207
+ packageMaterialization: WirePackageMaterialization | null | undefined,
208
+ ): { freshness: WireFreshness | null; schedule: string | null } {
209
+ const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
210
+ const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
211
+ const pkgLayer = packageFreshnessLayer(packageMaterialization);
212
+
213
+ const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
214
+ const fallback =
215
+ sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
216
+ const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
217
+
218
+ let freshness: WireFreshness | null = null;
219
+ if (window !== undefined || fallback !== undefined) {
220
+ freshness = {};
221
+ if (window !== undefined) freshness.window = window;
222
+ if (fallback !== undefined) freshness.fallback = fallback;
223
+ }
224
+ return { freshness, schedule };
225
+ }
226
+
97
227
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
98
228
  export function flattenDependsOn(node: {
99
229
  dependsOn: { sourceID: string }[];
@@ -291,6 +421,7 @@ export function deriveBuildPlan(
291
421
  connectionDigests: Record<string, string>,
292
422
  sourceNames?: string[],
293
423
  sourceModelPaths?: Record<string, string>,
424
+ packageMaterialization?: WirePackageMaterialization | null,
294
425
  ): BuildPlan {
295
426
  const include = sourceNames ? new Set(sourceNames) : null;
296
427
 
@@ -308,6 +439,15 @@ export function deriveBuildPlan(
308
439
  for (const [sourceID, source] of Object.entries(sources)) {
309
440
  if (include && !include.has(source.name)) continue;
310
441
  const annotationFields = deriveAnnotationFields(source);
442
+ // EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
443
+ // (source > model-file > package) and reported verbatim (null = unset at
444
+ // every level). Unlike `sharing`/`refresh` these are dotted/nested tag
445
+ // keys, so they come from resolveFreshnessSchedule rather than the scalar
446
+ // annotationFields map.
447
+ const { freshness, schedule } = resolveFreshnessSchedule(
448
+ source,
449
+ packageMaterialization,
450
+ );
311
451
  wireSources[sourceID] = {
312
452
  name: source.name,
313
453
  sourceID: source.sourceID,
@@ -323,6 +463,8 @@ export function deriveBuildPlan(
323
463
  // one that exists today.
324
464
  sharing: annotationFields.sharing ?? null,
325
465
  refresh: annotationFields.refresh ?? null,
466
+ freshness,
467
+ schedule,
326
468
  columns: deriveColumns(source),
327
469
  annotationFields,
328
470
  modelPath: sourceModelPaths?.[sourceID],
@@ -351,5 +493,6 @@ export async function computePackageBuildPlan(
351
493
  compiled.connectionDigests,
352
494
  undefined,
353
495
  compiled.sourceModelPaths,
496
+ pkg.getMaterializationConfig?.() ?? null,
354
497
  );
355
498
  }
@@ -7,17 +7,22 @@ import { Environment } from "./environment";
7
7
  import type { Package } from "./package";
8
8
 
9
9
  /**
10
- * The publish gate for the package-level materialization cron. At Phase B the
11
- * package-level cron (`materialization.schedule`) is replaced outright by
12
- * `materialization.freshness.window` (persistence.md §9.4), so ANY declared cron
13
- * is rejected regardless of the sources' sharing. The `sharing=private`
14
- * carve-out arrives whole with Phase A (persistence-phase-b-design.md §3.4);
15
- * until then there is no private artifact for a cron to own, so the B→A rule is
16
- * reject-all. These tests run a real `Environment` + `Package.create` over temp
17
- * dirs, so they also prove end-to-end that the pinned compiler ACCEPTS
18
- * `sharing=` / `refresh=` keys in the `#@ persist` annotation and that the
19
- * values survive to the wire build plan verbatim (unset stays null — never
20
- * defaulted to "shared").
10
+ * The publish gate for materialization crons (Phase A carve-out,
11
+ * persistence.md §9.4). A cron is the power tier of artifact-anchored
12
+ * scheduling and is valid only over `sharing="private"` artifacts:
13
+ *
14
+ * - a package-level cron (`materialization.schedule`) is valid only when
15
+ * EVERY governed persist source resolves to explicit `sharing="private"`;
16
+ * - a per-source cron (`#@ persist ... schedule=`) is valid only when THAT
17
+ * source resolves to explicit `sharing="private"`;
18
+ * - a cron on a shared/unset artifact is rejected, pointing at
19
+ * `freshness.window` as the shared-tier replacement.
20
+ *
21
+ * These tests run a real `Environment` + `Package.create` over temp dirs, so
22
+ * they also prove end-to-end that the pinned compiler ACCEPTS `sharing=` /
23
+ * `refresh=` / `schedule=` / `freshness.*` keys in the `#@ persist` annotation
24
+ * and that the values survive to the wire build plan verbatim (unset stays null
25
+ * — never defaulted to "shared").
21
26
  */
22
27
  describe("materialization cron gate", () => {
23
28
  let rootDir: string;
@@ -76,6 +81,20 @@ source: a is duckdb.sql("SELECT 1 as x")
76
81
 
77
82
  #@ persist name="b_table" sharing=private refresh=full
78
83
  source: b is duckdb.sql("SELECT 2 as x")
84
+ `;
85
+
86
+ // Per-source crons: valid on the private source, rejected on the shared and
87
+ // the unset (⇒ shared) sources.
88
+ const PER_SOURCE_CRON_MODEL = `##! experimental.persistence
89
+
90
+ #@ persist name="p_table" sharing=private schedule="0 */6 * * *"
91
+ source: p is duckdb.sql("SELECT 1 as x")
92
+
93
+ #@ persist name="s_table" sharing=shared schedule="0 0 * * *"
94
+ source: s is duckdb.sql("SELECT 2 as x")
95
+
96
+ #@ persist name="u_table" schedule="0 0 * * *"
97
+ source: u is duckdb.sql("SELECT 3 as x")
79
98
  `;
80
99
 
81
100
  it(
@@ -102,12 +121,13 @@ source: b is duckdb.sql("SELECT 2 as x")
102
121
  );
103
122
 
104
123
  it(
105
- "rejects any declared cron, pointing at freshness.window",
124
+ "rejects a package cron on a mixed package, pointing at freshness.window",
106
125
  async () => {
126
+ // Phase A: a package cron governs every persist source, so it is valid
127
+ // only when all resolve to explicit private. MIXED has a shared and an
128
+ // unset source, so the package cron is rejected.
107
129
  const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
108
130
 
109
- // One actionable message for the whole package — the B→A rule is
110
- // reject-all, not per-source.
111
131
  const warnings = pkg.scheduleWarnings();
112
132
  expect(warnings).toHaveLength(1);
113
133
  const joined = pkg.formatInvalidSchedule();
@@ -118,15 +138,34 @@ source: b is duckdb.sql("SELECT 2 as x")
118
138
  );
119
139
 
120
140
  it(
121
- "rejects a declared cron even when every persist source is private",
141
+ "accepts a package cron when every persist source is private",
122
142
  async () => {
123
- // The sharing=private carve-out arrives with Phase A; during B→A even
124
- // an all-private package's cron is rejected outright.
143
+ // Phase A carve-out: an all-private package legitimately carries a
144
+ // package-level cron (the power tier for its own private artifacts).
125
145
  const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
126
- expect(pkg.scheduleWarnings()).toHaveLength(1);
127
- expect(pkg.formatInvalidSchedule()).toContain(
128
- "materialization.freshness.window",
129
- );
146
+ expect(pkg.scheduleWarnings()).toEqual([]);
147
+ },
148
+ { timeout: 30000 },
149
+ );
150
+
151
+ it(
152
+ "rejects a per-source cron unless that source resolves to private",
153
+ async () => {
154
+ // No package cron here — only per-source `schedule=` declarations. The
155
+ // private source's cron is accepted; the shared and unset sources' are
156
+ // rejected, each pointing at freshness.window.
157
+ const pkg = await loadPackage(PER_SOURCE_CRON_MODEL);
158
+
159
+ const warnings = pkg.scheduleWarnings();
160
+ expect(warnings).toHaveLength(2);
161
+ const joined = warnings.join("\n");
162
+ expect(joined).toContain('"s"');
163
+ expect(joined).toContain('"u"');
164
+ expect(joined).not.toContain('"p"');
165
+ expect(joined).toContain("freshness.window");
166
+
167
+ // The resolved per-source schedule is surfaced verbatim on the plan.
168
+ expect(sourceByName(pkg, "p").schedule).toBe("0 */6 * * *");
130
169
  },
131
170
  { timeout: 30000 },
132
171
  );
@@ -921,6 +921,10 @@ describe("runBuild (branch behavior)", () => {
921
921
  sourceNames: string[] | undefined;
922
922
  forceRefresh: boolean;
923
923
  buildInstructions: BuildInstruction[] | undefined;
924
+ referenceManifest?:
925
+ | { sourceEntityId: string; physicalTableName: string }[]
926
+ | undefined;
927
+ strictUpstreams?: boolean | undefined;
924
928
  },
925
929
  signal: AbortSignal,
926
930
  ) => Promise<void>;
@@ -972,6 +976,44 @@ describe("runBuild (branch behavior)", () => {
972
976
  expect(svc.autoLoadManifest.called).toBe(false);
973
977
  });
974
978
 
979
+ it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
980
+ const svc = stubEngine();
981
+ const instructions = [makeInstruction()];
982
+
983
+ await svc.runBuild(
984
+ "mat-1",
985
+ "my-env",
986
+ "pkg",
987
+ {
988
+ sourceNames: undefined,
989
+ forceRefresh: false,
990
+ buildInstructions: instructions,
991
+ referenceManifest: [
992
+ { sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
993
+ ],
994
+ strictUpstreams: true,
995
+ },
996
+ new AbortController().signal,
997
+ );
998
+
999
+ // The reference manifest becomes the seed (carried) entries so a
1000
+ // downstream build resolves its upstream to the existing physical table.
1001
+ expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
1002
+ "up-1": {
1003
+ sourceEntityId: "up-1",
1004
+ physicalTableName: "upstream_tbl",
1005
+ },
1006
+ });
1007
+ // strictUpstreams flows through as the strict flag (5th arg).
1008
+ expect(svc.executeInstructedBuild.firstCall.args[4]).toBe(true);
1009
+ // Reused upstreams are counted as carried, not built.
1010
+ expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
1011
+ mode: "orchestrated",
1012
+ sourcesBuilt: 1,
1013
+ sourcesReused: 1,
1014
+ });
1015
+ });
1016
+
975
1017
  it("auto-run: derives instructions and auto-loads the manifest", async () => {
976
1018
  const svc = stubEngine();
977
1019
 
@@ -28,6 +28,7 @@ import {
28
28
  MaterializationStatus,
29
29
  MaterializationUpdate,
30
30
  ManifestEntry,
31
+ ManifestReference,
31
32
  ResourceRepository,
32
33
  } from "../storage/DatabaseInterface";
33
34
  import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
@@ -216,6 +217,8 @@ export class MaterializationService {
216
217
  forceRefresh?: boolean;
217
218
  sourceNames?: string[];
218
219
  buildInstructions?: BuildInstruction[];
220
+ referenceManifest?: ManifestReference[];
221
+ strictUpstreams?: boolean;
219
222
  } = {},
220
223
  ): Promise<Materialization> {
221
224
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -275,6 +278,8 @@ export class MaterializationService {
275
278
  sourceNames: options.sourceNames,
276
279
  forceRefresh,
277
280
  buildInstructions,
281
+ referenceManifest: options.referenceManifest,
282
+ strictUpstreams: options.strictUpstreams,
278
283
  },
279
284
  signal,
280
285
  ),
@@ -299,6 +304,8 @@ export class MaterializationService {
299
304
  sourceNames: string[] | undefined;
300
305
  forceRefresh: boolean;
301
306
  buildInstructions: BuildInstruction[] | undefined;
307
+ referenceManifest: ManifestReference[] | undefined;
308
+ strictUpstreams: boolean | undefined;
302
309
  },
303
310
  signal: AbortSignal,
304
311
  ): Promise<void> {
@@ -333,7 +340,12 @@ export class MaterializationService {
333
340
  let carried: Record<string, ManifestEntry>;
334
341
  if (orchestrated) {
335
342
  instructions = opts.buildInstructions!;
336
- carried = {};
343
+ // Seed the build Manifest with the caller-supplied upstream
344
+ // references so a downstream source's persist upstream (built in a
345
+ // prior run, not rebuilt here) resolves to its existing physical
346
+ // table instead of recomputing live. The reference key is the
347
+ // compiler's manifest-lookup sourceEntityId (see ManifestReference).
348
+ carried = this.referenceManifestToEntries(opts.referenceManifest);
337
349
  } else {
338
350
  // Skip-if-unchanged: reuse tables from the most recent successful
339
351
  // manifest for sources whose sourceEntityId is unchanged, unless
@@ -357,6 +369,7 @@ export class MaterializationService {
357
369
  instructions,
358
370
  carried,
359
371
  signal,
372
+ opts.strictUpstreams ?? false,
360
373
  );
361
374
 
362
375
  const sourcesBuilt = instructions.length;
@@ -450,6 +463,26 @@ export class MaterializationService {
450
463
  return { instructions, carried };
451
464
  }
452
465
 
466
+ /**
467
+ * Project the caller-supplied upstream reference manifest into the seed
468
+ * entry map `executeInstructedBuild` consumes. Each reference is keyed by the
469
+ * compiler's manifest-lookup sourceEntityId and carries only the physical
470
+ * table name — enough for the build Manifest to resolve a downstream persist
471
+ * reference to the existing table without rebuilding the upstream.
472
+ */
473
+ private referenceManifestToEntries(
474
+ referenceManifest: ManifestReference[] | undefined,
475
+ ): Record<string, ManifestEntry> {
476
+ const entries: Record<string, ManifestEntry> = {};
477
+ for (const ref of referenceManifest ?? []) {
478
+ entries[ref.sourceEntityId] = {
479
+ sourceEntityId: ref.sourceEntityId,
480
+ physicalTableName: ref.physicalTableName,
481
+ };
482
+ }
483
+ return entries;
484
+ }
485
+
453
486
  /**
454
487
  * Entries of the most recent successful (MANIFEST_FILE_READY) materialization
455
488
  * for this package, used for skip-if-unchanged. Excludes the in-flight run.
@@ -565,6 +598,7 @@ export class MaterializationService {
565
598
  instructions: BuildInstruction[],
566
599
  seedEntries: Record<string, ManifestEntry>,
567
600
  signal: AbortSignal,
601
+ strict = false,
568
602
  ): Promise<Record<string, ManifestEntry>> {
569
603
  const { graphs, sources, connectionDigests, connections } = compiled;
570
604
 
@@ -587,8 +621,12 @@ export class MaterializationService {
587
621
 
588
622
  // Accumulates physical names as sources are built so downstream sources
589
623
  // resolve their upstream references to the freshly-assigned tables. Seed
590
- // it with carried-forward entries so reused upstreams resolve too.
624
+ // it with carried-forward entries so reused upstreams resolve too. In
625
+ // strict mode, an upstream persist reference that is neither built here
626
+ // nor seeded fails the compile (runtime-manifest-strict-miss) instead of
627
+ // silently recomputing live.
591
628
  const manifest = new Manifest();
629
+ manifest.strict = strict;
592
630
  const entries: Record<string, ManifestEntry> = {};
593
631
  for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
594
632
  if (entry.physicalTableName) {
@@ -75,6 +75,41 @@ export function makeInstruction(
75
75
  * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
76
76
  * `#@ persist name=` annotation reader, defaulted to "unset").
77
77
  */
78
+ /** A freshness/schedule layer for the fake source's `#@` or `##` tag. */
79
+ interface FakeFreshnessSchedule {
80
+ freshness?: { window?: string; fallback?: string };
81
+ schedule?: string;
82
+ }
83
+
84
+ /**
85
+ * Build a fake Malloy `Tag` supporting both readers the build plan uses:
86
+ * `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
87
+ * and the path-based `text(...at)` (dotted `freshness.window` / `schedule`, for
88
+ * resolveFreshnessSchedule).
89
+ */
90
+ function fakeTag(
91
+ fields: Record<string, string> | undefined,
92
+ fs: FakeFreshnessSchedule | undefined,
93
+ ) {
94
+ return {
95
+ *entries() {
96
+ for (const [key, value] of Object.entries(fields ?? {})) {
97
+ yield [key, { text: () => value }];
98
+ }
99
+ },
100
+ text(...at: string[]): string | undefined {
101
+ if (at.length === 1) {
102
+ if (at[0] === "schedule") return fs?.schedule;
103
+ return fields?.[at[0]];
104
+ }
105
+ if (at.length === 2 && at[0] === "freshness") {
106
+ return fs?.freshness?.[at[1] as "window" | "fallback"];
107
+ }
108
+ return undefined;
109
+ },
110
+ };
111
+ }
112
+
78
113
  export function fakeSource(opts: {
79
114
  name: string;
80
115
  sourceEntityId: string;
@@ -83,6 +118,10 @@ export function fakeSource(opts: {
83
118
  dialectName?: string;
84
119
  /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
85
120
  annotationFields?: Record<string, string>;
121
+ /** Source-level (`#@`) freshness/schedule (dotted keys). */
122
+ freshnessSchedule?: FakeFreshnessSchedule;
123
+ /** Model-file-level (`##`) freshness/schedule default. */
124
+ modelFreshnessSchedule?: FakeFreshnessSchedule;
86
125
  }): PersistSource {
87
126
  const fields = opts.annotationFields;
88
127
  return {
@@ -93,20 +132,14 @@ export function fakeSource(opts: {
93
132
  makeBuildId: () => opts.sourceEntityId,
94
133
  getSQL: () => opts.sql ?? "SELECT 1",
95
134
  annotations: {
96
- parseAsTag: () =>
97
- fields
98
- ? {
99
- tag: {
100
- *entries() {
101
- for (const [key, value] of Object.entries(fields)) {
102
- yield [key, { text: () => value }];
103
- }
104
- },
105
- },
106
- }
107
- : // No annotation fields: entries() is absent, and
108
- // deriveAnnotationFields degrades to {}.
109
- { tag: { text: () => undefined } },
135
+ parseAsTag: () => ({
136
+ tag: fakeTag(fields, opts.freshnessSchedule),
137
+ }),
138
+ },
139
+ modelAnnotations: {
140
+ parseAsTag: () => ({
141
+ tag: fakeTag(undefined, opts.modelFreshnessSchedule),
142
+ }),
110
143
  },
111
144
  } as unknown as PersistSource;
112
145
  }