@malloy-publisher/server 0.0.223 → 0.0.225

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/app/api-doc.yaml +75 -0
  2. package/dist/app/assets/{EnvironmentPage-3wCdljov.js → EnvironmentPage-DYTeXDll.js} +1 -1
  3. package/dist/app/assets/{HomePage-K0GHwqq2.js → HomePage-pDK2BPJY.js} +1 -1
  4. package/dist/app/assets/{LightMode-CwU3kN4I.js → LightMode-C2bwGPY1.js} +1 -1
  5. package/dist/app/assets/{MainPage-CTncHE5T.js → MainPage-WtBulMH_.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-DziAOD6-.js → MaterializationsPage-hMgOtflG.js} +1 -1
  7. package/dist/app/assets/{ModelPage-XrS2jXEc.js → ModelPage-B2N5kYII.js} +1 -1
  8. package/dist/app/assets/{PackagePage-BkwgFxG8.js → PackagePage-CEN90nQG.js} +1 -1
  9. package/dist/app/assets/{RouteError-9cIQga6p.js → RouteError-BG2c5Zf0.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-DWiCRfEe.js → ThemeEditorPage-DNfeUwEZ.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-Ce7FM_Po.js → WorkbookPage-NKI1BhFS.js} +1 -1
  12. package/dist/app/assets/{core-D9Hl0IDY.es-CrO01m4X.js → core-C6anj5c0.es-DDLHqpzt.js} +1 -1
  13. package/dist/app/assets/{index-D2sm5RBA.js → index-C6gZ6sSY.js} +5 -5
  14. package/dist/app/assets/{index-CtQm7kvp.js → index-CzNqKMVl.js} +1 -1
  15. package/dist/app/assets/{index-wJU_kzl2.js → index-DMQtnaf4.js} +2 -2
  16. package/dist/app/assets/{index-Dz8hgkmy.js → index-JXgvyZna.js} +1 -1
  17. package/dist/app/assets/{index-8E2uLeV9.js → index-OEjKNSYb.js} +2 -2
  18. package/dist/app/index.html +1 -1
  19. package/dist/server.mjs +411 -26
  20. package/dist/sshcrypto-8m50vnmb.node +0 -0
  21. package/package.json +12 -12
  22. package/src/controller/materialization.controller.spec.ts +72 -0
  23. package/src/controller/materialization.controller.ts +75 -11
  24. package/src/mcp/skills/skills_bundle.json +1 -1
  25. package/src/service/build_plan.spec.ts +119 -0
  26. package/src/service/build_plan.ts +143 -0
  27. package/src/service/environment.ts +3 -3
  28. package/src/service/freshness.spec.ts +183 -0
  29. package/src/service/freshness.ts +112 -0
  30. package/src/service/manifest_loader.spec.ts +33 -0
  31. package/src/service/manifest_loader.ts +17 -8
  32. package/src/service/materialization_cron_gate.spec.ts +61 -17
  33. package/src/service/materialization_service.spec.ts +42 -0
  34. package/src/service/materialization_service.ts +46 -3
  35. package/src/service/materialization_test_fixtures.ts +47 -14
  36. package/src/service/model.ts +54 -4
  37. package/src/service/package.ts +173 -35
  38. package/src/storage/DatabaseInterface.ts +24 -0
  39. package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
  40. package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
  41. package/tests/fixtures/persist-multi-level/publisher.json +5 -0
  42. package/tests/integration/materialization/freshness_gate.integration.spec.ts +292 -0
  43. package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
@@ -10,6 +10,7 @@ import {
10
10
  deriveBuildPlan,
11
11
  flattenDependsOn,
12
12
  iterGraphSources,
13
+ resolveFreshnessSchedule,
13
14
  resolvePackageConnections,
14
15
  } from "./build_plan";
15
16
  import { fakeSource } from "./materialization_test_fixtures";
@@ -296,6 +297,124 @@ describe("deriveBuildPlan", () => {
296
297
  });
297
298
  });
298
299
 
300
+ describe("resolveFreshnessSchedule", () => {
301
+ it("reports source-level freshness + schedule verbatim", () => {
302
+ const source = fakeSource({
303
+ name: "s",
304
+ sourceEntityId: "bid",
305
+ freshnessSchedule: {
306
+ freshness: { window: "1h", fallback: "stale_ok" },
307
+ schedule: "0 */6 * * *",
308
+ },
309
+ });
310
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
311
+ freshness: { window: "1h", fallback: "stale_ok" },
312
+ schedule: "0 */6 * * *",
313
+ });
314
+ });
315
+
316
+ it("returns null for both when unset at every level", () => {
317
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
318
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
319
+ freshness: null,
320
+ schedule: null,
321
+ });
322
+ });
323
+
324
+ it("falls back to model-file then package per field (most-specific-wins)", () => {
325
+ // freshness.window from source, freshness.fallback from model-file. The
326
+ // package-level cron is NOT inherited as the source's effective schedule;
327
+ // schedule here comes only from the source's own model-file default.
328
+ const source = fakeSource({
329
+ name: "s",
330
+ sourceEntityId: "bid",
331
+ freshnessSchedule: { freshness: { window: "1h" } },
332
+ modelFreshnessSchedule: {
333
+ freshness: { fallback: "fail" },
334
+ schedule: "0 3 * * *",
335
+ },
336
+ });
337
+ const pkg = {
338
+ schedule: "0 0 * * *",
339
+ freshness: { window: "24h", fallback: "live" as const },
340
+ };
341
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
342
+ freshness: { window: "1h", fallback: "fail" },
343
+ schedule: "0 3 * * *",
344
+ });
345
+ });
346
+
347
+ it("does not inherit the package-level cron as the source's schedule", () => {
348
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
349
+ const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
350
+ // Freshness inherits from the package; schedule stays null (the package
351
+ // cron is gated separately, not folded into the per-source cron).
352
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
353
+ freshness: { window: "24h" },
354
+ schedule: null,
355
+ });
356
+ });
357
+
358
+ it("inherits the package freshness when the source and model are unset", () => {
359
+ const source = fakeSource({ name: "s", sourceEntityId: "bid" });
360
+ const pkg = { schedule: null, freshness: { window: "24h" } };
361
+ expect(resolveFreshnessSchedule(source, pkg)).toEqual({
362
+ freshness: { window: "24h" },
363
+ schedule: null,
364
+ });
365
+ });
366
+
367
+ it("drops an invalid fallback rather than defaulting it", () => {
368
+ const source = fakeSource({
369
+ name: "s",
370
+ sourceEntityId: "bid",
371
+ freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
372
+ });
373
+ expect(resolveFreshnessSchedule(source, null)).toEqual({
374
+ freshness: { window: "1h" },
375
+ schedule: null,
376
+ });
377
+ });
378
+ });
379
+
380
+ describe("deriveBuildPlan freshness/schedule", () => {
381
+ it("projects the resolved per-source freshness + schedule onto the plan", () => {
382
+ const source = fakeSource({
383
+ name: "s",
384
+ sourceEntityId: "bid",
385
+ annotationFields: { name: "s_table", sharing: "private" },
386
+ freshnessSchedule: {
387
+ freshness: { window: "1h" },
388
+ schedule: "0 */6 * * *",
389
+ },
390
+ });
391
+ const plan = deriveBuildPlan(
392
+ [
393
+ {
394
+ connectionName: "duckdb",
395
+ nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
396
+ },
397
+ ] as unknown as Parameters<typeof deriveBuildPlan>[0],
398
+ { "s@m": source },
399
+ { duckdb: "dig" },
400
+ undefined,
401
+ undefined,
402
+ {
403
+ schedule: null,
404
+ freshness: { window: "24h", fallback: "live" as const },
405
+ },
406
+ );
407
+
408
+ // Source window wins over the package default; package fallback fills the
409
+ // unset source fallback; source schedule reported verbatim.
410
+ expect(plan.sources["s@m"].freshness).toEqual({
411
+ window: "1h",
412
+ fallback: "live",
413
+ });
414
+ expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
415
+ });
416
+ });
417
+
299
418
  describe("compilePackageBuildPlan", () => {
300
419
  it("skips .malloynb notebooks without compiling them", async () => {
301
420
  // A notebook would throw on its `>>>` cell delimiter if compiled as a
@@ -17,6 +17,25 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
17
17
  type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
18
18
  type WireColumn = components["schemas"]["Column"];
19
19
  type BuildPlan = components["schemas"]["BuildPlan"];
20
+ type WireFreshness = components["schemas"]["Freshness"];
21
+ type WirePackageMaterialization =
22
+ components["schemas"]["PackageMaterializationConfig"];
23
+
24
+ /** The freshness `fallback` values the publisher recognizes; others are dropped. */
25
+ const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
26
+ type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
27
+
28
+ /** One layer's contribution to the resolved freshness/schedule (all optional). */
29
+ interface FreshnessScheduleLayer {
30
+ window?: string;
31
+ fallback?: FreshnessFallback;
32
+ schedule?: string;
33
+ }
34
+
35
+ /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
36
+ interface ReadableTag {
37
+ text(...path: string[]): string | undefined;
38
+ }
20
39
 
21
40
  /**
22
41
  * Minimal surface a package must expose to compile its build plan. Both
@@ -28,6 +47,13 @@ export interface BuildPlanPackage {
28
47
  getPackagePath(): string;
29
48
  getMalloyConfig(): MalloyConfig;
30
49
  getMalloyConnection(name: string): Promise<MalloyConnection>;
50
+ /**
51
+ * The package-level `materialization` config (from malloy-publisher.json),
52
+ * used as the least-specific layer when resolving per-source freshness /
53
+ * schedule. Optional so existing fixtures/callers that don't track it still
54
+ * typecheck (they resolve without a package default).
55
+ */
56
+ getMaterializationConfig?(): WirePackageMaterialization | null;
31
57
  }
32
58
 
33
59
  /**
@@ -94,6 +120,110 @@ export function deriveAnnotationFields(
94
120
  return out;
95
121
  }
96
122
 
123
+ /**
124
+ * Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
125
+ * `schedule`) from one Malloy tag layer, keeping only recognized values. These
126
+ * are dotted/nested tag properties, so they are NOT captured by the scalar
127
+ * {@link deriveAnnotationFields} loop and must be read by path here.
128
+ */
129
+ function tagFreshnessScheduleLayer(
130
+ tag: ReadableTag | undefined,
131
+ ): FreshnessScheduleLayer {
132
+ if (!tag || typeof tag.text !== "function") return {};
133
+ const layer: FreshnessScheduleLayer = {};
134
+ const window = tag.text("freshness", "window");
135
+ if (typeof window === "string") layer.window = window;
136
+ const fallback = tag.text("freshness", "fallback");
137
+ if (
138
+ typeof fallback === "string" &&
139
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
140
+ ) {
141
+ layer.fallback = fallback as FreshnessFallback;
142
+ }
143
+ const schedule = tag.text("schedule");
144
+ if (typeof schedule === "string") layer.schedule = schedule;
145
+ return layer;
146
+ }
147
+
148
+ /**
149
+ * The package-level `materialization` config as a freshness resolution layer.
150
+ * Only `freshness` inherits down to a source: the package-level cron
151
+ * (`materialization.schedule`) is a distinct package-grain concept surfaced on
152
+ * `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
153
+ * folded into a source's effective per-source cron (doing so would double-count
154
+ * against the package cron gate).
155
+ */
156
+ function packageFreshnessLayer(
157
+ cfg: WirePackageMaterialization | null | undefined,
158
+ ): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
159
+ if (!cfg) return {};
160
+ const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
161
+ if (cfg.freshness?.window) layer.window = cfg.freshness.window;
162
+ const fallback = cfg.freshness?.fallback;
163
+ if (
164
+ typeof fallback === "string" &&
165
+ (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
166
+ ) {
167
+ layer.fallback = fallback as FreshnessFallback;
168
+ }
169
+ return layer;
170
+ }
171
+
172
+ /** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
173
+ function safeSourceTag(source: PersistSource): ReadableTag | undefined {
174
+ try {
175
+ return source.annotations.parseAsTag("@").tag as ReadableTag;
176
+ } catch {
177
+ return undefined;
178
+ }
179
+ }
180
+
181
+ /**
182
+ * The model-file-level (`##`) tag resolved for the source, or undefined on a
183
+ * parse failure. The model-file layer sits between the source annotation and
184
+ * the package default in most-specific-wins resolution.
185
+ */
186
+ function safeModelTag(source: PersistSource): ReadableTag | undefined {
187
+ try {
188
+ return source.modelAnnotations.parseAsTag().tag as ReadableTag;
189
+ } catch {
190
+ return undefined;
191
+ }
192
+ }
193
+
194
+ /**
195
+ * Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
196
+ * at every level stays null so the control plane can distinguish "unset" —
197
+ * apply platform default — from an explicit declaration). Invalid `fallback`
198
+ * values are dropped, never defaulted. Precedence, per field, most-specific-wins:
199
+ *
200
+ * - `freshness`: source annotation > model-file default > package default.
201
+ * - `schedule`: source annotation > model-file default (the source's OWN cron).
202
+ * The package-level cron is a package-grain concept, surfaced and gated
203
+ * separately, so it is not folded into the per-source effective cron.
204
+ */
205
+ export function resolveFreshnessSchedule(
206
+ source: PersistSource,
207
+ packageMaterialization: WirePackageMaterialization | null | undefined,
208
+ ): { freshness: WireFreshness | null; schedule: string | null } {
209
+ const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
210
+ const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
211
+ const pkgLayer = packageFreshnessLayer(packageMaterialization);
212
+
213
+ const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
214
+ const fallback =
215
+ sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
216
+ const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
217
+
218
+ let freshness: WireFreshness | null = null;
219
+ if (window !== undefined || fallback !== undefined) {
220
+ freshness = {};
221
+ if (window !== undefined) freshness.window = window;
222
+ if (fallback !== undefined) freshness.fallback = fallback;
223
+ }
224
+ return { freshness, schedule };
225
+ }
226
+
97
227
  /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
98
228
  export function flattenDependsOn(node: {
99
229
  dependsOn: { sourceID: string }[];
@@ -291,6 +421,7 @@ export function deriveBuildPlan(
291
421
  connectionDigests: Record<string, string>,
292
422
  sourceNames?: string[],
293
423
  sourceModelPaths?: Record<string, string>,
424
+ packageMaterialization?: WirePackageMaterialization | null,
294
425
  ): BuildPlan {
295
426
  const include = sourceNames ? new Set(sourceNames) : null;
296
427
 
@@ -308,6 +439,15 @@ export function deriveBuildPlan(
308
439
  for (const [sourceID, source] of Object.entries(sources)) {
309
440
  if (include && !include.has(source.name)) continue;
310
441
  const annotationFields = deriveAnnotationFields(source);
442
+ // EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
443
+ // (source > model-file > package) and reported verbatim (null = unset at
444
+ // every level). Unlike `sharing`/`refresh` these are dotted/nested tag
445
+ // keys, so they come from resolveFreshnessSchedule rather than the scalar
446
+ // annotationFields map.
447
+ const { freshness, schedule } = resolveFreshnessSchedule(
448
+ source,
449
+ packageMaterialization,
450
+ );
311
451
  wireSources[sourceID] = {
312
452
  name: source.name,
313
453
  sourceID: source.sourceID,
@@ -323,6 +463,8 @@ export function deriveBuildPlan(
323
463
  // one that exists today.
324
464
  sharing: annotationFields.sharing ?? null,
325
465
  refresh: annotationFields.refresh ?? null,
466
+ freshness,
467
+ schedule,
326
468
  columns: deriveColumns(source),
327
469
  annotationFields,
328
470
  modelPath: sourceModelPaths?.[sourceID],
@@ -351,5 +493,6 @@ export async function computePackageBuildPlan(
351
493
  compiled.connectionDigests,
352
494
  undefined,
353
495
  compiled.sourceModelPaths,
496
+ pkg.getMaterializationConfig?.() ?? null,
354
497
  );
355
498
  }
@@ -28,7 +28,7 @@ import {
28
28
  assertSafeRelativeModelPath,
29
29
  safeJoinUnderRoot,
30
30
  } from "../path_safety";
31
- import { BuildManifest } from "../storage/DatabaseInterface";
31
+ import { FreshnessManifest } from "../storage/DatabaseInterface";
32
32
  import { URL_READER } from "../utils";
33
33
  import {
34
34
  buildEnvironmentMalloyConfig,
@@ -1119,7 +1119,7 @@ export class Environment {
1119
1119
  */
1120
1120
  public async reloadAllModelsForPackage(
1121
1121
  packageName: string,
1122
- manifest: BuildManifest["entries"],
1122
+ manifest: FreshnessManifest,
1123
1123
  ): Promise<void> {
1124
1124
  assertSafePackageName(packageName);
1125
1125
  return this.withPackageLock(packageName, async () => {
@@ -1193,7 +1193,7 @@ export class Environment {
1193
1193
  */
1194
1194
  private async fetchManifestEntriesWithTimeout(
1195
1195
  manifestLocation: string,
1196
- ): Promise<BuildManifest["entries"]> {
1196
+ ): Promise<FreshnessManifest> {
1197
1197
  let timer: ReturnType<typeof setTimeout> | undefined;
1198
1198
  const timeout = new Promise<never>((_, reject) => {
1199
1199
  timer = setTimeout(
@@ -0,0 +1,183 @@
1
+ import { describe, expect, it } from "bun:test";
2
+
3
+ import type {
4
+ FreshnessAwareManifestEntry,
5
+ FreshnessManifest,
6
+ } from "../storage/DatabaseInterface";
7
+ import { evaluateManifestFreshness, filterFreshManifest } from "./freshness";
8
+
9
+ // A fixed "now" so age math is deterministic.
10
+ const NOW = new Date("2026-07-07T12:00:00.000Z");
11
+ const NOW_MS = NOW.getTime();
12
+
13
+ /** dataAsOf `ageSeconds` before NOW. */
14
+ function asOf(ageSeconds: number): string {
15
+ return new Date(NOW_MS - ageSeconds * 1000).toISOString();
16
+ }
17
+
18
+ describe("evaluateManifestFreshness", () => {
19
+ it("serves the table when un-gated (no window)", () => {
20
+ const entry: FreshnessAwareManifestEntry = {
21
+ tableName: "t",
22
+ dataAsOf: asOf(1_000_000),
23
+ };
24
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
25
+ });
26
+
27
+ it("serves the table when a window is declared but no dataAsOf anchor", () => {
28
+ const entry: FreshnessAwareManifestEntry = {
29
+ tableName: "t",
30
+ freshnessWindowSeconds: 3600,
31
+ freshnessFallback: "live",
32
+ };
33
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
34
+ });
35
+
36
+ it("serves the table when fresh (age within window)", () => {
37
+ const entry: FreshnessAwareManifestEntry = {
38
+ tableName: "t",
39
+ dataAsOf: asOf(1800),
40
+ freshnessWindowSeconds: 3600,
41
+ freshnessFallback: "live",
42
+ };
43
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
44
+ });
45
+
46
+ it("serves the table at the exact window boundary (age == window)", () => {
47
+ const entry: FreshnessAwareManifestEntry = {
48
+ tableName: "t",
49
+ dataAsOf: asOf(3600),
50
+ freshnessWindowSeconds: 3600,
51
+ freshnessFallback: "live",
52
+ };
53
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
54
+ });
55
+
56
+ it("drops a stale entry with fallback live (=> serve live)", () => {
57
+ const entry: FreshnessAwareManifestEntry = {
58
+ tableName: "t",
59
+ dataAsOf: asOf(7200),
60
+ freshnessWindowSeconds: 3600,
61
+ freshnessFallback: "live",
62
+ };
63
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
64
+ });
65
+
66
+ it("keeps a stale entry with fallback stale_ok (=> serve the stale table)", () => {
67
+ const entry: FreshnessAwareManifestEntry = {
68
+ tableName: "t",
69
+ dataAsOf: asOf(7200),
70
+ freshnessWindowSeconds: 3600,
71
+ freshnessFallback: "stale_ok",
72
+ };
73
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
74
+ });
75
+
76
+ it("drops a stale entry with fallback fail (interim: serve live in v1)", () => {
77
+ const entry: FreshnessAwareManifestEntry = {
78
+ tableName: "t",
79
+ dataAsOf: asOf(7200),
80
+ freshnessWindowSeconds: 3600,
81
+ freshnessFallback: "fail",
82
+ };
83
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
84
+ });
85
+
86
+ it("drops a stale entry with no declared fallback (=> serve live)", () => {
87
+ const entry: FreshnessAwareManifestEntry = {
88
+ tableName: "t",
89
+ dataAsOf: asOf(7200),
90
+ freshnessWindowSeconds: 3600,
91
+ };
92
+ expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
93
+ });
94
+ });
95
+
96
+ describe("filterFreshManifest", () => {
97
+ it("includes fresh/un-gated/stale_ok and drops stale live/fail", () => {
98
+ const entries: FreshnessManifest = {
99
+ fresh: {
100
+ tableName: "fresh_t",
101
+ dataAsOf: asOf(1800),
102
+ freshnessWindowSeconds: 3600,
103
+ freshnessFallback: "live",
104
+ },
105
+ ungated: { tableName: "ungated_t", dataAsOf: asOf(999999) },
106
+ staleLive: {
107
+ tableName: "stale_live_t",
108
+ dataAsOf: asOf(7200),
109
+ freshnessWindowSeconds: 3600,
110
+ freshnessFallback: "live",
111
+ },
112
+ staleOk: {
113
+ tableName: "stale_ok_t",
114
+ dataAsOf: asOf(7200),
115
+ freshnessWindowSeconds: 3600,
116
+ freshnessFallback: "stale_ok",
117
+ },
118
+ staleFail: {
119
+ tableName: "stale_fail_t",
120
+ dataAsOf: asOf(7200),
121
+ freshnessWindowSeconds: 3600,
122
+ freshnessFallback: "fail",
123
+ },
124
+ };
125
+
126
+ const { manifest } = filterFreshManifest(entries, NOW);
127
+ expect(Object.keys(manifest).sort()).toEqual([
128
+ "fresh",
129
+ "staleOk",
130
+ "ungated",
131
+ ]);
132
+ expect(manifest.fresh).toEqual({ tableName: "fresh_t" });
133
+ expect(manifest.staleOk).toEqual({ tableName: "stale_ok_t" });
134
+ expect(manifest.ungated).toEqual({ tableName: "ungated_t" });
135
+ });
136
+
137
+ it("reports the earliest future window-crossing as nextStaleSince", () => {
138
+ const entries: FreshnessManifest = {
139
+ a: {
140
+ tableName: "a_t",
141
+ dataAsOf: asOf(1800), // window 3600 => flips at NOW + 1800s
142
+ freshnessWindowSeconds: 3600,
143
+ freshnessFallback: "live",
144
+ },
145
+ b: {
146
+ tableName: "b_t",
147
+ dataAsOf: asOf(1000), // window 3600 => flips at NOW + 2600s (later)
148
+ freshnessWindowSeconds: 3600,
149
+ freshnessFallback: "live",
150
+ },
151
+ };
152
+
153
+ const { nextStaleSince } = filterFreshManifest(entries, NOW);
154
+ // Earliest flip is entry a: dataAsOf(a) + window.
155
+ expect(nextStaleSince).toBe(NOW_MS - 1800 * 1000 + 3600 * 1000);
156
+ });
157
+
158
+ it("kept stale_ok entries do not set a future nextStaleSince", () => {
159
+ const entries: FreshnessManifest = {
160
+ staleOk: {
161
+ tableName: "stale_ok_t",
162
+ dataAsOf: asOf(7200),
163
+ freshnessWindowSeconds: 3600,
164
+ freshnessFallback: "stale_ok",
165
+ },
166
+ };
167
+ const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
168
+ expect(Object.keys(manifest)).toEqual(["staleOk"]);
169
+ // Its window crossing is already in the past, so nothing schedules a
170
+ // future recompute.
171
+ expect(nextStaleSince).toBeNull();
172
+ });
173
+
174
+ it("returns null nextStaleSince when nothing is gated", () => {
175
+ const entries: FreshnessManifest = {
176
+ a: { tableName: "a_t" },
177
+ b: { tableName: "b_t", dataAsOf: asOf(10) },
178
+ };
179
+ const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
180
+ expect(Object.keys(manifest).sort()).toEqual(["a", "b"]);
181
+ expect(nextStaleSince).toBeNull();
182
+ });
183
+ });
@@ -0,0 +1,112 @@
1
+ /**
2
+ * Per-query freshness gate for persisted sources (persistence.md §9.3, §14
3
+ * Phase B). A query on a `#@ persist` source may use its materialized table
4
+ * only while that table is within its declared freshness window; otherwise the
5
+ * query does what the source's `fallback` says.
6
+ *
7
+ * Two settled design principles anchor this (do not re-derive them the easy
8
+ * way):
9
+ *
10
+ * 1. Enforce per query, not at bind. Filtering stale entries once when the
11
+ * manifest is bound would let a table that crosses its window while the
12
+ * package stays loaded keep serving stale. The serve path re-evaluates
13
+ * freshness on every query.
14
+ * 2. Clock on `dataAsOf`, not build completion. The window is a deadline on
15
+ * *data* age; the anchor is the artifact's data-as-of instant (snapshot
16
+ * start) supplied by the control plane. `age = now - dataAsOf`.
17
+ *
18
+ * Scope: `live`/`stale_ok` are fully publisher-side. `fail` degrades to
19
+ * serve-live in this first cut (the entry is dropped like `live`); the airtight
20
+ * `fail` path — which must *error* a query that joins a stale persist source —
21
+ * is a gated forked-malloy follow-up and is not built here.
22
+ */
23
+
24
+ import {
25
+ BuildManifest,
26
+ FreshnessAwareManifestEntry,
27
+ FreshnessManifest,
28
+ } from "../storage/DatabaseInterface";
29
+
30
+ /**
31
+ * What to do with a single manifest entry at query time.
32
+ * - `serve_table`: substitute the materialized table (include `{ tableName }`).
33
+ * - `serve_live`: drop the entry so `strict:false` falls through to live SQL.
34
+ *
35
+ * `fail` collapses to `serve_live` in v1 (see module note).
36
+ */
37
+ export type FreshnessDecision = "serve_table" | "serve_live";
38
+
39
+ /**
40
+ * Evaluate one entry against `now`. An entry with no `freshnessWindowSeconds`
41
+ * is un-gated (always serve the table). A fresh entry (age within window) is
42
+ * served. A stale entry serves its table only under `stale_ok`; `live` and
43
+ * `fail` both drop to live in this first cut.
44
+ */
45
+ export function evaluateManifestFreshness(
46
+ entry: FreshnessAwareManifestEntry,
47
+ now: Date,
48
+ ): FreshnessDecision {
49
+ // Un-gated: no declared window ⇒ never stale. Also the rollout-safe default
50
+ // before the control plane starts stamping the freshness fields.
51
+ if (entry.freshnessWindowSeconds == null) return "serve_table";
52
+ // A window without an anchor can't be evaluated; treat as un-gated rather
53
+ // than tripping every query (matches "absent ⇒ never stale").
54
+ if (entry.dataAsOf == null) return "serve_table";
55
+ const dataAsOfMs = Date.parse(entry.dataAsOf);
56
+ if (Number.isNaN(dataAsOfMs)) return "serve_table";
57
+ const ageSeconds = (now.getTime() - dataAsOfMs) / 1000;
58
+ const stale = ageSeconds > entry.freshnessWindowSeconds;
59
+ if (!stale) return "serve_table";
60
+ return entry.freshnessFallback === "stale_ok" ? "serve_table" : "serve_live";
61
+ }
62
+
63
+ /**
64
+ * The instant a currently-fresh, gated entry will cross its window
65
+ * (`dataAsOf + window`), or `null` when the entry has no evaluable window
66
+ * (un-gated). Used to compute the next memo-invalidation instant.
67
+ */
68
+ function staleSinceMs(entry: FreshnessAwareManifestEntry): number | null {
69
+ if (entry.freshnessWindowSeconds == null || entry.dataAsOf == null) {
70
+ return null;
71
+ }
72
+ const dataAsOfMs = Date.parse(entry.dataAsOf);
73
+ if (Number.isNaN(dataAsOfMs)) return null;
74
+ return dataAsOfMs + entry.freshnessWindowSeconds * 1000;
75
+ }
76
+
77
+ /**
78
+ * Compute the freshness-filtered Malloy build manifest for `now` from the full
79
+ * retained wire entries.
80
+ *
81
+ * Returns the `sourceEntityId -> { tableName }` map of entries that should
82
+ * substitute their materialized table (fresh, un-gated, or stale-`stale_ok`),
83
+ * plus `nextStaleSince`: the earliest future instant a currently-included,
84
+ * gated entry will cross its window. Because age only ever grows, the
85
+ * fresh→stale transition is one-way, so a single `nextStaleSince` instant is a
86
+ * sufficient memo-invalidation deadline — no per-entry heap is needed. `null`
87
+ * means no included entry has an evaluable window, so the result never changes
88
+ * until the entries are replaced (rebind).
89
+ */
90
+ export function filterFreshManifest(
91
+ entries: FreshnessManifest,
92
+ now: Date,
93
+ ): { manifest: BuildManifest["entries"]; nextStaleSince: number | null } {
94
+ const manifest: BuildManifest["entries"] = {};
95
+ let nextStaleSince: number | null = null;
96
+ const nowMs = now.getTime();
97
+
98
+ for (const [sourceEntityId, entry] of Object.entries(entries)) {
99
+ if (evaluateManifestFreshness(entry, now) === "serve_live") continue;
100
+ manifest[sourceEntityId] = { tableName: entry.tableName };
101
+
102
+ // Only currently-fresh gated entries can flip later; a kept stale
103
+ // `stale_ok` entry (staleSince already in the past) never changes again.
104
+ const flipAt = staleSinceMs(entry);
105
+ if (flipAt != null && flipAt > nowMs) {
106
+ nextStaleSince =
107
+ nextStaleSince == null ? flipAt : Math.min(nextStaleSince, flipAt);
108
+ }
109
+ }
110
+
111
+ return { manifest, nextStaleSince };
112
+ }
@@ -46,6 +46,39 @@ describe("fetchManifestEntries", () => {
46
46
  });
47
47
  });
48
48
 
49
+ it("carries the control-plane freshness fields verbatim", async () => {
50
+ const dataAsOf = "2026-07-07T00:00:00.000Z";
51
+ const file = await writeManifest({
52
+ entries: {
53
+ gated: {
54
+ sourceEntityId: "gated",
55
+ physicalTableName: "schema.gated_mz",
56
+ dataAsOf,
57
+ freshnessWindowSeconds: 3600,
58
+ freshnessFallback: "stale_ok",
59
+ },
60
+ ungated: {
61
+ sourceEntityId: "ungated",
62
+ physicalTableName: "schema.ungated_mz",
63
+ },
64
+ },
65
+ });
66
+
67
+ const entries = await fetchManifestEntries(file);
68
+
69
+ // Filter-free: freshness fields are retained for the serve-path gate; a
70
+ // window-less entry is left un-gated.
71
+ expect(entries).toEqual({
72
+ gated: {
73
+ tableName: "schema.gated_mz",
74
+ dataAsOf,
75
+ freshnessWindowSeconds: 3600,
76
+ freshnessFallback: "stale_ok",
77
+ },
78
+ ungated: { tableName: "schema.ungated_mz" },
79
+ });
80
+ });
81
+
49
82
  it("reads via a file:// URI", async () => {
50
83
  const file = await writeManifest({
51
84
  entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },