@malloy-publisher/server 0.0.224 → 0.0.226

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/app/api-doc.yaml +76 -13
  2. package/dist/app/assets/{EnvironmentPage-DYTeXDll.js → EnvironmentPage-DvOJ7L_b.js} +1 -1
  3. package/dist/app/assets/{HomePage-pDK2BPJY.js → HomePage-CXguJsXS.js} +1 -1
  4. package/dist/app/assets/{LightMode-C2bwGPY1.js → LightMode-ZsshUznu.js} +1 -1
  5. package/dist/app/assets/{MainPage-WtBulMH_.js → MainPage-BIe0VwBa.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-hMgOtflG.js → MaterializationsPage-BuZ6UJVx.js} +1 -1
  7. package/dist/app/assets/{ModelPage-B2N5kYII.js → ModelPage-DsPf-s8B.js} +1 -1
  8. package/dist/app/assets/{PackagePage-CEN90nQG.js → PackagePage-CEVNAKZa.js} +1 -1
  9. package/dist/app/assets/{RouteError-BG2c5Zf0.js → RouteError-Chn7lL96.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-DNfeUwEZ.js → ThemeEditorPage-DWC_FdNU.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-NKI1BhFS.js → WorkbookPage-CGrsFz8p.js} +1 -1
  12. package/dist/app/assets/{core-C6anj5c0.es-DDLHqpzt.js → core-vVgoO8IR.es-BD_THWs_.js} +1 -1
  13. package/dist/app/assets/{index-DMQtnaf4.js → index-BioohWQj.js} +1 -1
  14. package/dist/app/assets/{index-CzNqKMVl.js → index-D6YtyiJ0.js} +1 -1
  15. package/dist/app/assets/{index-C6gZ6sSY.js → index-DNUZpnaa.js} +4 -4
  16. package/dist/app/assets/{index-JXgvyZna.js → index-gEWxu09x.js} +1 -1
  17. package/dist/app/index.html +1 -1
  18. package/dist/instrumentation.mjs +71 -15
  19. package/dist/package_load_worker.mjs +83 -16
  20. package/dist/server.mjs +249 -46
  21. package/dist/sshcrypto-8m50vnmb.node +0 -0
  22. package/package.json +1 -1
  23. package/src/controller/materialization.controller.spec.ts +72 -0
  24. package/src/controller/materialization.controller.ts +75 -11
  25. package/src/controller/package.controller.spec.ts +17 -17
  26. package/src/controller/package.controller.ts +7 -6
  27. package/src/logger.spec.ts +193 -0
  28. package/src/logger.ts +115 -18
  29. package/src/mcp/skills/skills_bundle.json +1 -1
  30. package/src/package_load/package_load_pool.ts +5 -1
  31. package/src/package_load/package_load_worker.ts +7 -0
  32. package/src/package_load/protocol.ts +5 -1
  33. package/src/service/build_plan.spec.ts +110 -9
  34. package/src/service/build_plan.ts +126 -7
  35. package/src/service/environment.ts +4 -0
  36. package/src/service/materialization_service.spec.ts +42 -0
  37. package/src/service/materialization_service.ts +40 -2
  38. package/src/service/materialization_test_fixtures.ts +46 -15
  39. package/src/service/package.ts +116 -32
  40. package/src/service/package_manifest.spec.ts +22 -1
  41. package/src/service/package_manifest.ts +29 -0
  42. package/src/service/persistence_policy.spec.ts +234 -0
  43. package/src/storage/DatabaseInterface.ts +1 -0
  44. package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
  45. package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
  46. package/tests/fixtures/persist-multi-level/publisher.json +5 -0
  47. package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
  48. package/src/service/materialization_cron_gate.spec.ts +0 -142
@@ -81,7 +81,10 @@ import { fileURLToPath, pathToFileURL } from "url";
81
81
 
82
82
  import { ModelCompilationError } from "../errors";
83
83
  import { logger } from "../logger";
84
- import type { PackageMaterializationConfig } from "../service/package_manifest";
84
+ import type {
85
+ PackageMaterializationConfig,
86
+ PackageScope,
87
+ } from "../service/package_manifest";
85
88
  import type {
86
89
  ConnectionMetadataRequest,
87
90
  ConnectionMetadataResponse,
@@ -235,6 +238,7 @@ export interface LoadPackageOutcome {
235
238
  queryableSources?: "declared" | "all";
236
239
  manifestLocation?: string | null;
237
240
  materialization?: PackageMaterializationConfig | null;
241
+ scope?: PackageScope;
238
242
  };
239
243
  models: Array<
240
244
  Omit<SerializedModel, "modelDef" | "sourceInfos"> & {
@@ -85,7 +85,9 @@ import { validateAuthorizeProbes } from "../service/authorize";
85
85
  import { type FilterDefinition } from "../service/filter";
86
86
  import {
87
87
  PackageMaterializationConfig,
88
+ PackageScope,
88
89
  parsePackageMaterialization,
90
+ parsePackageScope,
89
91
  } from "../service/package_manifest";
90
92
  import {
91
93
  extractQueriesFromModelDef,
@@ -387,6 +389,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
387
389
  queryableSources?: "declared" | "all";
388
390
  manifestLocation?: string | null;
389
391
  materialization?: PackageMaterializationConfig | null;
392
+ scope?: PackageScope;
390
393
  }> {
391
394
  const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
392
395
  const contents = await fs.promises.readFile(manifestPath, "utf8");
@@ -397,6 +400,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
397
400
  queryableSources?: unknown;
398
401
  manifestLocation?: unknown;
399
402
  materialization?: unknown;
403
+ scope?: unknown;
400
404
  };
401
405
  return {
402
406
  name: parsed.name,
@@ -416,6 +420,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
416
420
  // Package-level Malloy Persistence policy; surfaced to the control plane,
417
421
  // which owns scheduling. Only `schedule` is read today.
418
422
  materialization: parsePackageMaterialization(parsed.materialization),
423
+ // Package-level persist scope mode; defaults to "package". An invalid
424
+ // value throws here and fails the load (scope is load-bearing).
425
+ scope: parsePackageScope(parsed.scope),
419
426
  };
420
427
  }
421
428
 
@@ -65,7 +65,10 @@
65
65
  */
66
66
 
67
67
  import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
68
- import type { PackageMaterializationConfig } from "../service/package_manifest";
68
+ import type {
69
+ PackageMaterializationConfig,
70
+ PackageScope,
71
+ } from "../service/package_manifest";
69
72
 
70
73
  // ──────────────────────────────────────────────────────────────────────
71
74
  // Direction: main ──▶ worker (load-package job)
@@ -181,6 +184,7 @@ export interface LoadPackageResult {
181
184
  queryableSources?: "declared" | "all";
182
185
  manifestLocation?: string | null;
183
186
  materialization?: PackageMaterializationConfig | null;
187
+ scope?: PackageScope;
184
188
  };
185
189
  models: SerializedModel[];
186
190
  /** Wall-clock ms inside the worker for the full package load. */
@@ -10,6 +10,7 @@ import {
10
10
  deriveBuildPlan,
11
11
  flattenDependsOn,
12
12
  iterGraphSources,
13
+ resolveFreshness,
13
14
  resolvePackageConnections,
14
15
  } from "./build_plan";
15
16
  import { fakeSource } from "./materialization_test_fixtures";
@@ -212,10 +213,10 @@ describe("deriveBuildPlan", () => {
212
213
  });
213
214
  });
214
215
 
215
- it("reports declared sharing/refresh verbatim and null when unset", () => {
216
- // The control plane distinguishes unset from an explicit `shared` (it
217
- // applies the platform default itself), so the publisher must report
218
- // the declared value verbatim never substitute the default.
216
+ it("reports declared refresh verbatim (null when unset) and does not emit sharing/schedule", () => {
217
+ // `refresh` is a metadata pass-through; `sharing`/`schedule` were retired
218
+ // from the contract and must not be emitted as typed fields (they stay in
219
+ // the raw annotationFields for the publish-time validator to detect).
219
220
  const declared = fakeSource({
220
221
  name: "declared",
221
222
  sourceEntityId: "bid-d",
@@ -242,17 +243,22 @@ describe("deriveBuildPlan", () => {
242
243
  { duckdb: "dig" },
243
244
  );
244
245
 
245
- expect(plan.sources["declared@m"].sharing).toBe("private");
246
246
  expect(plan.sources["declared@m"].refresh).toBe("incremental");
247
- // The raw annotation map still carries every field alongside the typed
248
- // projections.
247
+ // Retired typed fields are absent from the wire projection.
248
+ expect(
249
+ (plan.sources["declared@m"] as Record<string, unknown>).sharing,
250
+ ).toBeUndefined();
251
+ expect(
252
+ (plan.sources["declared@m"] as Record<string, unknown>).schedule,
253
+ ).toBeUndefined();
254
+ // The raw annotation map still carries every field (so the validator can
255
+ // reject a source-level sharing/schedule at publish).
249
256
  expect(plan.sources["declared@m"].annotationFields).toEqual({
250
257
  name: "d_table",
251
258
  sharing: "private",
252
259
  refresh: "incremental",
253
260
  });
254
- // Unset is null — not "shared" — on the wire.
255
- expect(plan.sources["unset@m"].sharing).toBeNull();
261
+ // Unset refresh is null on the wire.
256
262
  expect(plan.sources["unset@m"].refresh).toBeNull();
257
263
  });
258
264
 
@@ -296,6 +302,101 @@ describe("deriveBuildPlan", () => {
296
302
  });
297
303
  });
298
304
 
305
+ describe("resolveFreshness", () => {
306
+ it("reports source-level freshness verbatim", () => {
307
+ const source = fakeSource({
308
+ name: "s",
309
+ sourceEntityId: "bid",
310
+ freshnessSchedule: {
311
+ freshness: { window: "1h", fallback: "stale_ok" },
312
+ },
313
+ });
314
+ expect(resolveFreshness(source, null)).toEqual({
315
+ window: "1h",
316
+ fallback: "stale_ok",
317
+ });
318
+ });
319
+
320
+ it("returns null when unset at every level", () => {
321
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
322
+ expect(resolveFreshness(source, null)).toBeNull();
323
+ });
324
+
325
+ it("falls back to model-file then package per field (most-specific-wins)", () => {
326
+ // freshness.window from source, freshness.fallback from model-file.
327
+ const source = fakeSource({
328
+ name: "s",
329
+ sourceEntityId: "bid",
330
+ freshnessSchedule: { freshness: { window: "1h" } },
331
+ modelFreshnessSchedule: { freshness: { fallback: "fail" } },
332
+ });
333
+ const pkg = {
334
+ schedule: null,
335
+ freshness: { window: "24h", fallback: "live" as const },
336
+ };
337
+ expect(resolveFreshness(source, pkg)).toEqual({
338
+ window: "1h",
339
+ fallback: "fail",
340
+ });
341
+ });
342
+
343
+ it("inherits the package freshness when the source and model are unset", () => {
344
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
345
+ const pkg = { schedule: null, freshness: { window: "24h" } };
346
+ expect(resolveFreshness(source, pkg)).toEqual({ window: "24h" });
347
+ });
348
+
349
+ it("drops an invalid fallback rather than defaulting it", () => {
350
+ const source = fakeSource({
351
+ name: "s",
352
+ sourceEntityId: "bid",
353
+ freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
354
+ });
355
+ expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
356
+ });
357
+ });
358
+
359
+ describe("deriveBuildPlan freshness", () => {
360
+ it("projects the resolved per-source freshness onto the plan (no schedule/sharing)", () => {
361
+ const source = fakeSource({
362
+ name: "s",
363
+ sourceEntityId: "bid",
364
+ annotationFields: { name: "s_table" },
365
+ freshnessSchedule: { freshness: { window: "1h" } },
366
+ });
367
+ const plan = deriveBuildPlan(
368
+ [
369
+ {
370
+ connectionName: "duckdb",
371
+ nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
372
+ },
373
+ ] as unknown as Parameters<typeof deriveBuildPlan>[0],
374
+ { "s@m": source },
375
+ { duckdb: "dig" },
376
+ undefined,
377
+ undefined,
378
+ {
379
+ schedule: null,
380
+ freshness: { window: "24h", fallback: "live" as const },
381
+ },
382
+ );
383
+
384
+ // Source window wins over the package default; package fallback fills the
385
+ // unset source fallback.
386
+ expect(plan.sources["s@m"].freshness).toEqual({
387
+ window: "1h",
388
+ fallback: "live",
389
+ });
390
+ // Retired fields are not emitted.
391
+ expect(
392
+ (plan.sources["s@m"] as Record<string, unknown>).schedule,
393
+ ).toBeUndefined();
394
+ expect(
395
+ (plan.sources["s@m"] as Record<string, unknown>).sharing,
396
+ ).toBeUndefined();
397
+ });
398
+ });
399
+
299
400
  describe("compilePackageBuildPlan", () => {
300
401
  it("skips .malloynb notebooks without compiling them", async () => {
301
402
  // A notebook would throw on its `>>>` cell delimiter if compiled as a
@@ -17,6 +17,24 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
17
17
  type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
18
18
  type WireColumn = components["schemas"]["Column"];
19
19
  type BuildPlan = components["schemas"]["BuildPlan"];
20
+ type WireFreshness = components["schemas"]["Freshness"];
21
+ type WirePackageMaterialization =
22
+ components["schemas"]["PackageMaterializationConfig"];
23
+
24
+ /** The freshness `fallback` values the publisher recognizes; others are dropped. */
25
+ const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
26
+ type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
27
+
28
+ /** One layer's contribution to the resolved freshness (all optional). */
29
+ interface FreshnessLayer {
30
+ window?: string;
31
+ fallback?: FreshnessFallback;
32
+ }
33
+
34
+ /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
35
+ interface ReadableTag {
36
+ text(...path: string[]): string | undefined;
37
+ }
20
38
 
21
39
  /**
22
40
  * Minimal surface a package must expose to compile its build plan. Both
@@ -28,6 +46,13 @@ export interface BuildPlanPackage {
28
46
  getPackagePath(): string;
29
47
  getMalloyConfig(): MalloyConfig;
30
48
  getMalloyConnection(name: string): Promise<MalloyConnection>;
49
+ /**
50
+ * The package-level `materialization` config (from malloy-publisher.json),
51
+ * used as the least-specific layer when resolving per-source freshness /
52
+ * schedule. Optional so existing fixtures/callers that don't track it still
53
+ * typecheck (they resolve without a package default).
54
+ */
55
+ getMaterializationConfig?(): WirePackageMaterialization | null;
31
56
  }
32
57
 
33
58
  /**
@@ -94,6 +119,96 @@ export function deriveAnnotationFields(
94
119
  return out;
95
120
  }
96
121
 
122
+ /**
123
+ * Read the freshness keys (`freshness.window`, `freshness.fallback`) from one
124
+ * Malloy tag layer, keeping only recognized values. These are dotted/nested tag
125
+ * properties, so they are NOT captured by the scalar {@link deriveAnnotationFields}
126
+ * loop and must be read by path here. (Per-source `sharing`/`schedule` were
127
+ * retired — scope is package-level and a schedule is package-root-only — so they
128
+ * are not resolved here; declaring either on a source is a publish-time manifest
129
+ * error, enforced in Package.persistencePolicyWarnings.)
130
+ */
131
+ function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
132
+ if (!tag || typeof tag.text !== "function") return {};
133
+ const layer: FreshnessLayer = {};
134
+ const window = tag.text("freshness", "window");
135
+ if (typeof window === "string") layer.window = window;
136
+ const fallback = tag.text("freshness", "fallback");
137
+ if (
138
+ typeof fallback === "string" &&
139
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
140
+ ) {
141
+ layer.fallback = fallback as FreshnessFallback;
142
+ }
143
+ return layer;
144
+ }
145
+
146
+ /** The package-level `materialization.freshness` as a resolution layer. */
147
+ function packageFreshnessLayer(
148
+ cfg: WirePackageMaterialization | null | undefined,
149
+ ): FreshnessLayer {
150
+ if (!cfg) return {};
151
+ const layer: FreshnessLayer = {};
152
+ if (cfg.freshness?.window) layer.window = cfg.freshness.window;
153
+ const fallback = cfg.freshness?.fallback;
154
+ if (
155
+ typeof fallback === "string" &&
156
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
157
+ ) {
158
+ layer.fallback = fallback as FreshnessFallback;
159
+ }
160
+ return layer;
161
+ }
162
+
163
+ /** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
164
+ function safeSourceTag(source: PersistSource): ReadableTag | undefined {
165
+ try {
166
+ return source.annotations.parseAsTag("@").tag as ReadableTag;
167
+ } catch {
168
+ return undefined;
169
+ }
170
+ }
171
+
172
+ /**
173
+ * The model-file-level (`##`) tag resolved for the source, or undefined on a
174
+ * parse failure. The model-file layer sits between the source annotation and
175
+ * the package default in most-specific-wins resolution.
176
+ */
177
+ function safeModelTag(source: PersistSource): ReadableTag | undefined {
178
+ try {
179
+ return source.modelAnnotations.parseAsTag().tag as ReadableTag;
180
+ } catch {
181
+ return undefined;
182
+ }
183
+ }
184
+
185
+ /**
186
+ * Resolve a source's EFFECTIVE freshness with most-specific-wins precedence,
187
+ * per field: source annotation > model-file default > package default. Reported
188
+ * verbatim (unset at every level stays null so the control plane can
189
+ * distinguish "unset" — apply platform default — from an explicit declaration).
190
+ * Invalid `fallback` values are dropped, never defaulted. Freshness is valid in
191
+ * both scope modes.
192
+ */
193
+ export function resolveFreshness(
194
+ source: PersistSource,
195
+ packageMaterialization: WirePackageMaterialization | null | undefined,
196
+ ): WireFreshness | null {
197
+ const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
198
+ const modelLayer = tagFreshnessLayer(safeModelTag(source));
199
+ const pkgLayer = packageFreshnessLayer(packageMaterialization);
200
+
201
+ const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
202
+ const fallback =
203
+ sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
204
+
205
+ if (window === undefined && fallback === undefined) return null;
206
+ const freshness: WireFreshness = {};
207
+ if (window !== undefined) freshness.window = window;
208
+ if (fallback !== undefined) freshness.fallback = fallback;
209
+ return freshness;
210
+ }
211
+
97
212
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
98
213
  export function flattenDependsOn(node: {
99
214
  dependsOn: { sourceID: string }[];
@@ -291,6 +406,7 @@ export function deriveBuildPlan(
291
406
  connectionDigests: Record<string, string>,
292
407
  sourceNames?: string[],
293
408
  sourceModelPaths?: Record<string, string>,
409
+ packageMaterialization?: WirePackageMaterialization | null,
294
410
  ): BuildPlan {
295
411
  const include = sourceNames ? new Set(sourceNames) : null;
296
412
 
@@ -308,6 +424,14 @@ export function deriveBuildPlan(
308
424
  for (const [sourceID, source] of Object.entries(sources)) {
309
425
  if (include && !include.has(source.name)) continue;
310
426
  const annotationFields = deriveAnnotationFields(source);
427
+ // EFFECTIVE per-source freshness, resolved most-specific-wins
428
+ // (source > model-file > package) and reported verbatim (null = unset at
429
+ // every level). Freshness is a dotted/nested tag key, so it comes from
430
+ // resolveFreshness rather than the scalar annotationFields map, and is
431
+ // valid in both scope modes. Per-source `sharing`/`schedule` are NOT
432
+ // emitted (retired from the contract); if a source declares either it is
433
+ // rejected at publish (Package.persistencePolicyWarnings) — the raw keys
434
+ // still ride `annotationFields` so the validator can detect them.
311
435
  wireSources[sourceID] = {
312
436
  name: source.name,
313
437
  sourceID: source.sourceID,
@@ -315,14 +439,8 @@ export function deriveBuildPlan(
315
439
  dialect: source.dialectName,
316
440
  sourceEntityId: computeSourceEntityId(source, connectionDigests),
317
441
  sql: source.getSQL(),
318
- // Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
319
- // = unset). The control plane applies the platform default (unset =>
320
- // shared) itself, so the publisher must never substitute it — unset
321
- // has to stay distinguishable from an explicit `shared`. The source's
322
- // own annotation is the most-specific declaration layer, and the only
323
- // one that exists today.
324
- sharing: annotationFields.sharing ?? null,
325
442
  refresh: annotationFields.refresh ?? null,
443
+ freshness: resolveFreshness(source, packageMaterialization),
326
444
  columns: deriveColumns(source),
327
445
  annotationFields,
328
446
  modelPath: sourceModelPaths?.[sourceID],
@@ -351,5 +469,6 @@ export async function computePackageBuildPlan(
351
469
  compiled.connectionDigests,
352
470
  undefined,
353
471
  compiled.sourceModelPaths,
472
+ pkg.getMaterializationConfig?.() ?? null,
354
473
  );
355
474
  }
@@ -1347,6 +1347,10 @@ export class Environment {
1347
1347
  // control plane misread the gap as a schedule removal. It is not a
1348
1348
  // PATCH-editable field, so always preserve the existing value.
1349
1349
  materialization: existing.materialization,
1350
+ // Same rationale for the manifest-derived persist `scope`: not
1351
+ // PATCH-editable, so preserve it rather than dropping it to the
1352
+ // default until the next reload.
1353
+ scope: existing.scope,
1350
1354
  });
1351
1355
 
1352
1356
  // Strict-reject, symmetric with the publish path
@@ -921,6 +921,10 @@ describe("runBuild (branch behavior)", () => {
921
921
  sourceNames: string[] | undefined;
922
922
  forceRefresh: boolean;
923
923
  buildInstructions: BuildInstruction[] | undefined;
924
+ referenceManifest?:
925
+ | { sourceEntityId: string; physicalTableName: string }[]
926
+ | undefined;
927
+ strictUpstreams?: boolean | undefined;
924
928
  },
925
929
  signal: AbortSignal,
926
930
  ) => Promise<void>;
@@ -972,6 +976,44 @@ describe("runBuild (branch behavior)", () => {
972
976
  expect(svc.autoLoadManifest.called).toBe(false);
973
977
  });
974
978
 
979
+ it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
980
+ const svc = stubEngine();
981
+ const instructions = [makeInstruction()];
982
+
983
+ await svc.runBuild(
984
+ "mat-1",
985
+ "my-env",
986
+ "pkg",
987
+ {
988
+ sourceNames: undefined,
989
+ forceRefresh: false,
990
+ buildInstructions: instructions,
991
+ referenceManifest: [
992
+ { sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
993
+ ],
994
+ strictUpstreams: true,
995
+ },
996
+ new AbortController().signal,
997
+ );
998
+
999
+ // The reference manifest becomes the seed (carried) entries so a
1000
+ // downstream build resolves its upstream to the existing physical table.
1001
+ expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
1002
+ "up-1": {
1003
+ sourceEntityId: "up-1",
1004
+ physicalTableName: "upstream_tbl",
1005
+ },
1006
+ });
1007
+ // strictUpstreams flows through as the strict flag (5th arg).
1008
+ expect(svc.executeInstructedBuild.firstCall.args[4]).toBe(true);
1009
+ // Reused upstreams are counted as carried, not built.
1010
+ expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
1011
+ mode: "orchestrated",
1012
+ sourcesBuilt: 1,
1013
+ sourcesReused: 1,
1014
+ });
1015
+ });
1016
+
975
1017
  it("auto-run: derives instructions and auto-loads the manifest", async () => {
976
1018
  const svc = stubEngine();
977
1019
 
@@ -28,6 +28,7 @@ import {
28
28
  MaterializationStatus,
29
29
  MaterializationUpdate,
30
30
  ManifestEntry,
31
+ ManifestReference,
31
32
  ResourceRepository,
32
33
  } from "../storage/DatabaseInterface";
33
34
  import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
@@ -216,6 +217,8 @@ export class MaterializationService {
216
217
  forceRefresh?: boolean;
217
218
  sourceNames?: string[];
218
219
  buildInstructions?: BuildInstruction[];
220
+ referenceManifest?: ManifestReference[];
221
+ strictUpstreams?: boolean;
219
222
  } = {},
220
223
  ): Promise<Materialization> {
221
224
  const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -275,6 +278,8 @@ export class MaterializationService {
275
278
  sourceNames: options.sourceNames,
276
279
  forceRefresh,
277
280
  buildInstructions,
281
+ referenceManifest: options.referenceManifest,
282
+ strictUpstreams: options.strictUpstreams,
278
283
  },
279
284
  signal,
280
285
  ),
@@ -299,6 +304,8 @@ export class MaterializationService {
299
304
  sourceNames: string[] | undefined;
300
305
  forceRefresh: boolean;
301
306
  buildInstructions: BuildInstruction[] | undefined;
307
+ referenceManifest: ManifestReference[] | undefined;
308
+ strictUpstreams: boolean | undefined;
302
309
  },
303
310
  signal: AbortSignal,
304
311
  ): Promise<void> {
@@ -333,7 +340,12 @@ export class MaterializationService {
333
340
  let carried: Record<string, ManifestEntry>;
334
341
  if (orchestrated) {
335
342
  instructions = opts.buildInstructions!;
336
- carried = {};
343
+ // Seed the build Manifest with the caller-supplied upstream
344
+ // references so a downstream source's persist upstream (built in a
345
+ // prior run, not rebuilt here) resolves to its existing physical
346
+ // table instead of recomputing live. The reference key is the
347
+ // compiler's manifest-lookup sourceEntityId (see ManifestReference).
348
+ carried = this.referenceManifestToEntries(opts.referenceManifest);
337
349
  } else {
338
350
  // Skip-if-unchanged: reuse tables from the most recent successful
339
351
  // manifest for sources whose sourceEntityId is unchanged, unless
@@ -357,6 +369,7 @@ export class MaterializationService {
357
369
  instructions,
358
370
  carried,
359
371
  signal,
372
+ opts.strictUpstreams ?? false,
360
373
  );
361
374
 
362
375
  const sourcesBuilt = instructions.length;
@@ -450,6 +463,26 @@ export class MaterializationService {
450
463
  return { instructions, carried };
451
464
  }
452
465
 
466
+ /**
467
+ * Project the caller-supplied upstream reference manifest into the seed
468
+ * entry map `executeInstructedBuild` consumes. Each reference is keyed by the
469
+ * compiler's manifest-lookup sourceEntityId and carries only the physical
470
+ * table name — enough for the build Manifest to resolve a downstream persist
471
+ * reference to the existing table without rebuilding the upstream.
472
+ */
473
+ private referenceManifestToEntries(
474
+ referenceManifest: ManifestReference[] | undefined,
475
+ ): Record<string, ManifestEntry> {
476
+ const entries: Record<string, ManifestEntry> = {};
477
+ for (const ref of referenceManifest ?? []) {
478
+ entries[ref.sourceEntityId] = {
479
+ sourceEntityId: ref.sourceEntityId,
480
+ physicalTableName: ref.physicalTableName,
481
+ };
482
+ }
483
+ return entries;
484
+ }
485
+
453
486
  /**
454
487
  * Entries of the most recent successful (MANIFEST_FILE_READY) materialization
455
488
  * for this package, used for skip-if-unchanged. Excludes the in-flight run.
@@ -565,6 +598,7 @@ export class MaterializationService {
565
598
  instructions: BuildInstruction[],
566
599
  seedEntries: Record<string, ManifestEntry>,
567
600
  signal: AbortSignal,
601
+ strict = false,
568
602
  ): Promise<Record<string, ManifestEntry>> {
569
603
  const { graphs, sources, connectionDigests, connections } = compiled;
570
604
 
@@ -587,8 +621,12 @@ export class MaterializationService {
587
621
 
588
622
  // Accumulates physical names as sources are built so downstream sources
589
623
  // resolve their upstream references to the freshly-assigned tables. Seed
590
- // it with carried-forward entries so reused upstreams resolve too.
624
+ // it with carried-forward entries so reused upstreams resolve too. In
625
+ // strict mode, an upstream persist reference that is neither built here
626
+ // nor seeded fails the compile (runtime-manifest-strict-miss) instead of
627
+ // silently recomputing live.
591
628
  const manifest = new Manifest();
629
+ manifest.strict = strict;
592
630
  const entries: Record<string, ManifestEntry> = {};
593
631
  for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
594
632
  if (entry.physicalTableName) {
@@ -75,14 +75,51 @@ export function makeInstruction(
75
75
  * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
76
76
  * `#@ persist name=` annotation reader, defaulted to "unset").
77
77
  */
78
+ /** A freshness layer for the fake source's `#@` or `##` tag. */
79
+ interface FakeFreshnessSchedule {
80
+ freshness?: { window?: string; fallback?: string };
81
+ }
82
+
83
+ /**
84
+ * Build a fake Malloy `Tag` supporting both readers the build plan uses:
85
+ * `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
86
+ * and the path-based `text(...at)` (dotted `freshness.window`, for
87
+ * resolveFreshness).
88
+ */
89
+ function fakeTag(
90
+ fields: Record<string, string> | undefined,
91
+ fs: FakeFreshnessSchedule | undefined,
92
+ ) {
93
+ return {
94
+ *entries() {
95
+ for (const [key, value] of Object.entries(fields ?? {})) {
96
+ yield [key, { text: () => value }];
97
+ }
98
+ },
99
+ text(...at: string[]): string | undefined {
100
+ if (at.length === 1) {
101
+ return fields?.[at[0]];
102
+ }
103
+ if (at.length === 2 && at[0] === "freshness") {
104
+ return fs?.freshness?.[at[1] as "window" | "fallback"];
105
+ }
106
+ return undefined;
107
+ },
108
+ };
109
+ }
110
+
78
111
  export function fakeSource(opts: {
79
112
  name: string;
80
113
  sourceEntityId: string;
81
114
  sql?: string;
82
115
  connectionName?: string;
83
116
  dialectName?: string;
84
- /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
117
+ /** key=value fields of the `#@ persist` annotation (e.g. name, refresh). */
85
118
  annotationFields?: Record<string, string>;
119
+ /** Source-level (`#@`) freshness (dotted keys). */
120
+ freshnessSchedule?: FakeFreshnessSchedule;
121
+ /** Model-file-level (`##`) freshness default. */
122
+ modelFreshnessSchedule?: FakeFreshnessSchedule;
86
123
  }): PersistSource {
87
124
  const fields = opts.annotationFields;
88
125
  return {
@@ -93,20 +130,14 @@ export function fakeSource(opts: {
93
130
  makeBuildId: () => opts.sourceEntityId,
94
131
  getSQL: () => opts.sql ?? "SELECT 1",
95
132
  annotations: {
96
- parseAsTag: () =>
97
- fields
98
- ? {
99
- tag: {
100
- *entries() {
101
- for (const [key, value] of Object.entries(fields)) {
102
- yield [key, { text: () => value }];
103
- }
104
- },
105
- },
106
- }
107
- : // No annotation fields: entries() is absent, and
108
- // deriveAnnotationFields degrades to {}.
109
- { tag: { text: () => undefined } },
133
+ parseAsTag: () => ({
134
+ tag: fakeTag(fields, opts.freshnessSchedule),
135
+ }),
136
+ },
137
+ modelAnnotations: {
138
+ parseAsTag: () => ({
139
+ tag: fakeTag(undefined, opts.modelFreshnessSchedule),
140
+ }),
110
141
  },
111
142
  } as unknown as PersistSource;
112
143
  }