@malloy-publisher/server 0.0.223 → 0.0.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +75 -0
- package/dist/app/assets/{EnvironmentPage-3wCdljov.js → EnvironmentPage-DYTeXDll.js} +1 -1
- package/dist/app/assets/{HomePage-K0GHwqq2.js → HomePage-pDK2BPJY.js} +1 -1
- package/dist/app/assets/{LightMode-CwU3kN4I.js → LightMode-C2bwGPY1.js} +1 -1
- package/dist/app/assets/{MainPage-CTncHE5T.js → MainPage-WtBulMH_.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-DziAOD6-.js → MaterializationsPage-hMgOtflG.js} +1 -1
- package/dist/app/assets/{ModelPage-XrS2jXEc.js → ModelPage-B2N5kYII.js} +1 -1
- package/dist/app/assets/{PackagePage-BkwgFxG8.js → PackagePage-CEN90nQG.js} +1 -1
- package/dist/app/assets/{RouteError-9cIQga6p.js → RouteError-BG2c5Zf0.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DWiCRfEe.js → ThemeEditorPage-DNfeUwEZ.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Ce7FM_Po.js → WorkbookPage-NKI1BhFS.js} +1 -1
- package/dist/app/assets/{core-D9Hl0IDY.es-CrO01m4X.js → core-C6anj5c0.es-DDLHqpzt.js} +1 -1
- package/dist/app/assets/{index-D2sm5RBA.js → index-C6gZ6sSY.js} +5 -5
- package/dist/app/assets/{index-CtQm7kvp.js → index-CzNqKMVl.js} +1 -1
- package/dist/app/assets/{index-wJU_kzl2.js → index-DMQtnaf4.js} +2 -2
- package/dist/app/assets/{index-Dz8hgkmy.js → index-JXgvyZna.js} +1 -1
- package/dist/app/assets/{index-8E2uLeV9.js → index-OEjKNSYb.js} +2 -2
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +411 -26
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +12 -12
- package/src/controller/materialization.controller.spec.ts +72 -0
- package/src/controller/materialization.controller.ts +75 -11
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/service/build_plan.spec.ts +119 -0
- package/src/service/build_plan.ts +143 -0
- package/src/service/environment.ts +3 -3
- package/src/service/freshness.spec.ts +183 -0
- package/src/service/freshness.ts +112 -0
- package/src/service/manifest_loader.spec.ts +33 -0
- package/src/service/manifest_loader.ts +17 -8
- package/src/service/materialization_cron_gate.spec.ts +61 -17
- package/src/service/materialization_service.spec.ts +42 -0
- package/src/service/materialization_service.ts +46 -3
- package/src/service/materialization_test_fixtures.ts +47 -14
- package/src/service/model.ts +54 -4
- package/src/service/package.ts +173 -35
- package/src/storage/DatabaseInterface.ts +24 -0
- package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
- package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
- package/tests/fixtures/persist-multi-level/publisher.json +5 -0
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +292 -0
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
|
@@ -10,6 +10,7 @@ import {
|
|
|
10
10
|
deriveBuildPlan,
|
|
11
11
|
flattenDependsOn,
|
|
12
12
|
iterGraphSources,
|
|
13
|
+
resolveFreshnessSchedule,
|
|
13
14
|
resolvePackageConnections,
|
|
14
15
|
} from "./build_plan";
|
|
15
16
|
import { fakeSource } from "./materialization_test_fixtures";
|
|
@@ -296,6 +297,124 @@ describe("deriveBuildPlan", () => {
|
|
|
296
297
|
});
|
|
297
298
|
});
|
|
298
299
|
|
|
300
|
+
describe("resolveFreshnessSchedule", () => {
|
|
301
|
+
it("reports source-level freshness + schedule verbatim", () => {
|
|
302
|
+
const source = fakeSource({
|
|
303
|
+
name: "s",
|
|
304
|
+
sourceEntityId: "bid",
|
|
305
|
+
freshnessSchedule: {
|
|
306
|
+
freshness: { window: "1h", fallback: "stale_ok" },
|
|
307
|
+
schedule: "0 */6 * * *",
|
|
308
|
+
},
|
|
309
|
+
});
|
|
310
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
311
|
+
freshness: { window: "1h", fallback: "stale_ok" },
|
|
312
|
+
schedule: "0 */6 * * *",
|
|
313
|
+
});
|
|
314
|
+
});
|
|
315
|
+
|
|
316
|
+
it("returns null for both when unset at every level", () => {
|
|
317
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
318
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
319
|
+
freshness: null,
|
|
320
|
+
schedule: null,
|
|
321
|
+
});
|
|
322
|
+
});
|
|
323
|
+
|
|
324
|
+
it("falls back to model-file then package per field (most-specific-wins)", () => {
|
|
325
|
+
// freshness.window from source, freshness.fallback from model-file. The
|
|
326
|
+
// package-level cron is NOT inherited as the source's effective schedule;
|
|
327
|
+
// schedule here comes only from the source's own model-file default.
|
|
328
|
+
const source = fakeSource({
|
|
329
|
+
name: "s",
|
|
330
|
+
sourceEntityId: "bid",
|
|
331
|
+
freshnessSchedule: { freshness: { window: "1h" } },
|
|
332
|
+
modelFreshnessSchedule: {
|
|
333
|
+
freshness: { fallback: "fail" },
|
|
334
|
+
schedule: "0 3 * * *",
|
|
335
|
+
},
|
|
336
|
+
});
|
|
337
|
+
const pkg = {
|
|
338
|
+
schedule: "0 0 * * *",
|
|
339
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
340
|
+
};
|
|
341
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
342
|
+
freshness: { window: "1h", fallback: "fail" },
|
|
343
|
+
schedule: "0 3 * * *",
|
|
344
|
+
});
|
|
345
|
+
});
|
|
346
|
+
|
|
347
|
+
it("does not inherit the package-level cron as the source's schedule", () => {
|
|
348
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
349
|
+
const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
|
|
350
|
+
// Freshness inherits from the package; schedule stays null (the package
|
|
351
|
+
// cron is gated separately, not folded into the per-source cron).
|
|
352
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
353
|
+
freshness: { window: "24h" },
|
|
354
|
+
schedule: null,
|
|
355
|
+
});
|
|
356
|
+
});
|
|
357
|
+
|
|
358
|
+
it("inherits the package freshness when the source and model are unset", () => {
|
|
359
|
+
const source = fakeSource({ name: "s", sourceEntityId: "bid" });
|
|
360
|
+
const pkg = { schedule: null, freshness: { window: "24h" } };
|
|
361
|
+
expect(resolveFreshnessSchedule(source, pkg)).toEqual({
|
|
362
|
+
freshness: { window: "24h" },
|
|
363
|
+
schedule: null,
|
|
364
|
+
});
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
it("drops an invalid fallback rather than defaulting it", () => {
|
|
368
|
+
const source = fakeSource({
|
|
369
|
+
name: "s",
|
|
370
|
+
sourceEntityId: "bid",
|
|
371
|
+
freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
|
|
372
|
+
});
|
|
373
|
+
expect(resolveFreshnessSchedule(source, null)).toEqual({
|
|
374
|
+
freshness: { window: "1h" },
|
|
375
|
+
schedule: null,
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
});
|
|
379
|
+
|
|
380
|
+
describe("deriveBuildPlan freshness/schedule", () => {
|
|
381
|
+
it("projects the resolved per-source freshness + schedule onto the plan", () => {
|
|
382
|
+
const source = fakeSource({
|
|
383
|
+
name: "s",
|
|
384
|
+
sourceEntityId: "bid",
|
|
385
|
+
annotationFields: { name: "s_table", sharing: "private" },
|
|
386
|
+
freshnessSchedule: {
|
|
387
|
+
freshness: { window: "1h" },
|
|
388
|
+
schedule: "0 */6 * * *",
|
|
389
|
+
},
|
|
390
|
+
});
|
|
391
|
+
const plan = deriveBuildPlan(
|
|
392
|
+
[
|
|
393
|
+
{
|
|
394
|
+
connectionName: "duckdb",
|
|
395
|
+
nodes: [[{ sourceID: "s@m", dependsOn: [] }]],
|
|
396
|
+
},
|
|
397
|
+
] as unknown as Parameters<typeof deriveBuildPlan>[0],
|
|
398
|
+
{ "s@m": source },
|
|
399
|
+
{ duckdb: "dig" },
|
|
400
|
+
undefined,
|
|
401
|
+
undefined,
|
|
402
|
+
{
|
|
403
|
+
schedule: null,
|
|
404
|
+
freshness: { window: "24h", fallback: "live" as const },
|
|
405
|
+
},
|
|
406
|
+
);
|
|
407
|
+
|
|
408
|
+
// Source window wins over the package default; package fallback fills the
|
|
409
|
+
// unset source fallback; source schedule reported verbatim.
|
|
410
|
+
expect(plan.sources["s@m"].freshness).toEqual({
|
|
411
|
+
window: "1h",
|
|
412
|
+
fallback: "live",
|
|
413
|
+
});
|
|
414
|
+
expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
|
|
415
|
+
});
|
|
416
|
+
});
|
|
417
|
+
|
|
299
418
|
describe("compilePackageBuildPlan", () => {
|
|
300
419
|
it("skips .malloynb notebooks without compiling them", async () => {
|
|
301
420
|
// A notebook would throw on its `>>>` cell delimiter if compiled as a
|
|
@@ -17,6 +17,25 @@ type WireBuildGraph = components["schemas"]["BuildGraph"];
|
|
|
17
17
|
type WirePersistSourcePlan = components["schemas"]["PersistSourcePlan"];
|
|
18
18
|
type WireColumn = components["schemas"]["Column"];
|
|
19
19
|
type BuildPlan = components["schemas"]["BuildPlan"];
|
|
20
|
+
type WireFreshness = components["schemas"]["Freshness"];
|
|
21
|
+
type WirePackageMaterialization =
|
|
22
|
+
components["schemas"]["PackageMaterializationConfig"];
|
|
23
|
+
|
|
24
|
+
/** The freshness `fallback` values the publisher recognizes; others are dropped. */
|
|
25
|
+
const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
|
|
26
|
+
type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
|
|
27
|
+
|
|
28
|
+
/** One layer's contribution to the resolved freshness/schedule (all optional). */
|
|
29
|
+
interface FreshnessScheduleLayer {
|
|
30
|
+
window?: string;
|
|
31
|
+
fallback?: FreshnessFallback;
|
|
32
|
+
schedule?: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
|
|
36
|
+
interface ReadableTag {
|
|
37
|
+
text(...path: string[]): string | undefined;
|
|
38
|
+
}
|
|
20
39
|
|
|
21
40
|
/**
|
|
22
41
|
* Minimal surface a package must expose to compile its build plan. Both
|
|
@@ -28,6 +47,13 @@ export interface BuildPlanPackage {
|
|
|
28
47
|
getPackagePath(): string;
|
|
29
48
|
getMalloyConfig(): MalloyConfig;
|
|
30
49
|
getMalloyConnection(name: string): Promise<MalloyConnection>;
|
|
50
|
+
/**
|
|
51
|
+
* The package-level `materialization` config (from malloy-publisher.json),
|
|
52
|
+
* used as the least-specific layer when resolving per-source freshness /
|
|
53
|
+
* schedule. Optional so existing fixtures/callers that don't track it still
|
|
54
|
+
* typecheck (they resolve without a package default).
|
|
55
|
+
*/
|
|
56
|
+
getMaterializationConfig?(): WirePackageMaterialization | null;
|
|
31
57
|
}
|
|
32
58
|
|
|
33
59
|
/**
|
|
@@ -94,6 +120,110 @@ export function deriveAnnotationFields(
|
|
|
94
120
|
return out;
|
|
95
121
|
}
|
|
96
122
|
|
|
123
|
+
/**
|
|
124
|
+
* Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
|
|
125
|
+
* `schedule`) from one Malloy tag layer, keeping only recognized values. These
|
|
126
|
+
* are dotted/nested tag properties, so they are NOT captured by the scalar
|
|
127
|
+
* {@link deriveAnnotationFields} loop and must be read by path here.
|
|
128
|
+
*/
|
|
129
|
+
function tagFreshnessScheduleLayer(
|
|
130
|
+
tag: ReadableTag | undefined,
|
|
131
|
+
): FreshnessScheduleLayer {
|
|
132
|
+
if (!tag || typeof tag.text !== "function") return {};
|
|
133
|
+
const layer: FreshnessScheduleLayer = {};
|
|
134
|
+
const window = tag.text("freshness", "window");
|
|
135
|
+
if (typeof window === "string") layer.window = window;
|
|
136
|
+
const fallback = tag.text("freshness", "fallback");
|
|
137
|
+
if (
|
|
138
|
+
typeof fallback === "string" &&
|
|
139
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
140
|
+
) {
|
|
141
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
142
|
+
}
|
|
143
|
+
const schedule = tag.text("schedule");
|
|
144
|
+
if (typeof schedule === "string") layer.schedule = schedule;
|
|
145
|
+
return layer;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* The package-level `materialization` config as a freshness resolution layer.
|
|
150
|
+
* Only `freshness` inherits down to a source: the package-level cron
|
|
151
|
+
* (`materialization.schedule`) is a distinct package-grain concept surfaced on
|
|
152
|
+
* `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
|
|
153
|
+
* folded into a source's effective per-source cron (doing so would double-count
|
|
154
|
+
* against the package cron gate).
|
|
155
|
+
*/
|
|
156
|
+
function packageFreshnessLayer(
|
|
157
|
+
cfg: WirePackageMaterialization | null | undefined,
|
|
158
|
+
): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
|
|
159
|
+
if (!cfg) return {};
|
|
160
|
+
const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
|
|
161
|
+
if (cfg.freshness?.window) layer.window = cfg.freshness.window;
|
|
162
|
+
const fallback = cfg.freshness?.fallback;
|
|
163
|
+
if (
|
|
164
|
+
typeof fallback === "string" &&
|
|
165
|
+
(FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
|
|
166
|
+
) {
|
|
167
|
+
layer.fallback = fallback as FreshnessFallback;
|
|
168
|
+
}
|
|
169
|
+
return layer;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The source's `#@` tag, or undefined if it fails to parse (degrade to unset). */
|
|
173
|
+
function safeSourceTag(source: PersistSource): ReadableTag | undefined {
|
|
174
|
+
try {
|
|
175
|
+
return source.annotations.parseAsTag("@").tag as ReadableTag;
|
|
176
|
+
} catch {
|
|
177
|
+
return undefined;
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* The model-file-level (`##`) tag resolved for the source, or undefined on a
|
|
183
|
+
* parse failure. The model-file layer sits between the source annotation and
|
|
184
|
+
* the package default in most-specific-wins resolution.
|
|
185
|
+
*/
|
|
186
|
+
function safeModelTag(source: PersistSource): ReadableTag | undefined {
|
|
187
|
+
try {
|
|
188
|
+
return source.modelAnnotations.parseAsTag().tag as ReadableTag;
|
|
189
|
+
} catch {
|
|
190
|
+
return undefined;
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/**
|
|
195
|
+
* Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
|
|
196
|
+
* at every level stays null so the control plane can distinguish "unset" —
|
|
197
|
+
* apply platform default — from an explicit declaration). Invalid `fallback`
|
|
198
|
+
* values are dropped, never defaulted. Precedence, per field, most-specific-wins:
|
|
199
|
+
*
|
|
200
|
+
* - `freshness`: source annotation > model-file default > package default.
|
|
201
|
+
* - `schedule`: source annotation > model-file default (the source's OWN cron).
|
|
202
|
+
* The package-level cron is a package-grain concept, surfaced and gated
|
|
203
|
+
* separately, so it is not folded into the per-source effective cron.
|
|
204
|
+
*/
|
|
205
|
+
export function resolveFreshnessSchedule(
|
|
206
|
+
source: PersistSource,
|
|
207
|
+
packageMaterialization: WirePackageMaterialization | null | undefined,
|
|
208
|
+
): { freshness: WireFreshness | null; schedule: string | null } {
|
|
209
|
+
const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
|
|
210
|
+
const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
|
|
211
|
+
const pkgLayer = packageFreshnessLayer(packageMaterialization);
|
|
212
|
+
|
|
213
|
+
const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
|
|
214
|
+
const fallback =
|
|
215
|
+
sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
|
|
216
|
+
const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
|
|
217
|
+
|
|
218
|
+
let freshness: WireFreshness | null = null;
|
|
219
|
+
if (window !== undefined || fallback !== undefined) {
|
|
220
|
+
freshness = {};
|
|
221
|
+
if (window !== undefined) freshness.window = window;
|
|
222
|
+
if (fallback !== undefined) freshness.fallback = fallback;
|
|
223
|
+
}
|
|
224
|
+
return { freshness, schedule };
|
|
225
|
+
}
|
|
226
|
+
|
|
97
227
|
/** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
|
|
98
228
|
export function flattenDependsOn(node: {
|
|
99
229
|
dependsOn: { sourceID: string }[];
|
|
@@ -291,6 +421,7 @@ export function deriveBuildPlan(
|
|
|
291
421
|
connectionDigests: Record<string, string>,
|
|
292
422
|
sourceNames?: string[],
|
|
293
423
|
sourceModelPaths?: Record<string, string>,
|
|
424
|
+
packageMaterialization?: WirePackageMaterialization | null,
|
|
294
425
|
): BuildPlan {
|
|
295
426
|
const include = sourceNames ? new Set(sourceNames) : null;
|
|
296
427
|
|
|
@@ -308,6 +439,15 @@ export function deriveBuildPlan(
|
|
|
308
439
|
for (const [sourceID, source] of Object.entries(sources)) {
|
|
309
440
|
if (include && !include.has(source.name)) continue;
|
|
310
441
|
const annotationFields = deriveAnnotationFields(source);
|
|
442
|
+
// EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
|
|
443
|
+
// (source > model-file > package) and reported verbatim (null = unset at
|
|
444
|
+
// every level). Unlike `sharing`/`refresh` these are dotted/nested tag
|
|
445
|
+
// keys, so they come from resolveFreshnessSchedule rather than the scalar
|
|
446
|
+
// annotationFields map.
|
|
447
|
+
const { freshness, schedule } = resolveFreshnessSchedule(
|
|
448
|
+
source,
|
|
449
|
+
packageMaterialization,
|
|
450
|
+
);
|
|
311
451
|
wireSources[sourceID] = {
|
|
312
452
|
name: source.name,
|
|
313
453
|
sourceID: source.sourceID,
|
|
@@ -323,6 +463,8 @@ export function deriveBuildPlan(
|
|
|
323
463
|
// one that exists today.
|
|
324
464
|
sharing: annotationFields.sharing ?? null,
|
|
325
465
|
refresh: annotationFields.refresh ?? null,
|
|
466
|
+
freshness,
|
|
467
|
+
schedule,
|
|
326
468
|
columns: deriveColumns(source),
|
|
327
469
|
annotationFields,
|
|
328
470
|
modelPath: sourceModelPaths?.[sourceID],
|
|
@@ -351,5 +493,6 @@ export async function computePackageBuildPlan(
|
|
|
351
493
|
compiled.connectionDigests,
|
|
352
494
|
undefined,
|
|
353
495
|
compiled.sourceModelPaths,
|
|
496
|
+
pkg.getMaterializationConfig?.() ?? null,
|
|
354
497
|
);
|
|
355
498
|
}
|
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
assertSafeRelativeModelPath,
|
|
29
29
|
safeJoinUnderRoot,
|
|
30
30
|
} from "../path_safety";
|
|
31
|
-
import {
|
|
31
|
+
import { FreshnessManifest } from "../storage/DatabaseInterface";
|
|
32
32
|
import { URL_READER } from "../utils";
|
|
33
33
|
import {
|
|
34
34
|
buildEnvironmentMalloyConfig,
|
|
@@ -1119,7 +1119,7 @@ export class Environment {
|
|
|
1119
1119
|
*/
|
|
1120
1120
|
public async reloadAllModelsForPackage(
|
|
1121
1121
|
packageName: string,
|
|
1122
|
-
manifest:
|
|
1122
|
+
manifest: FreshnessManifest,
|
|
1123
1123
|
): Promise<void> {
|
|
1124
1124
|
assertSafePackageName(packageName);
|
|
1125
1125
|
return this.withPackageLock(packageName, async () => {
|
|
@@ -1193,7 +1193,7 @@ export class Environment {
|
|
|
1193
1193
|
*/
|
|
1194
1194
|
private async fetchManifestEntriesWithTimeout(
|
|
1195
1195
|
manifestLocation: string,
|
|
1196
|
-
): Promise<
|
|
1196
|
+
): Promise<FreshnessManifest> {
|
|
1197
1197
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1198
1198
|
const timeout = new Promise<never>((_, reject) => {
|
|
1199
1199
|
timer = setTimeout(
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
FreshnessAwareManifestEntry,
|
|
5
|
+
FreshnessManifest,
|
|
6
|
+
} from "../storage/DatabaseInterface";
|
|
7
|
+
import { evaluateManifestFreshness, filterFreshManifest } from "./freshness";
|
|
8
|
+
|
|
9
|
+
// A fixed "now" so age math is deterministic.
|
|
10
|
+
const NOW = new Date("2026-07-07T12:00:00.000Z");
|
|
11
|
+
const NOW_MS = NOW.getTime();
|
|
12
|
+
|
|
13
|
+
/** dataAsOf `ageSeconds` before NOW. */
|
|
14
|
+
function asOf(ageSeconds: number): string {
|
|
15
|
+
return new Date(NOW_MS - ageSeconds * 1000).toISOString();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe("evaluateManifestFreshness", () => {
|
|
19
|
+
it("serves the table when un-gated (no window)", () => {
|
|
20
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
21
|
+
tableName: "t",
|
|
22
|
+
dataAsOf: asOf(1_000_000),
|
|
23
|
+
};
|
|
24
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("serves the table when a window is declared but no dataAsOf anchor", () => {
|
|
28
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
29
|
+
tableName: "t",
|
|
30
|
+
freshnessWindowSeconds: 3600,
|
|
31
|
+
freshnessFallback: "live",
|
|
32
|
+
};
|
|
33
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("serves the table when fresh (age within window)", () => {
|
|
37
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
38
|
+
tableName: "t",
|
|
39
|
+
dataAsOf: asOf(1800),
|
|
40
|
+
freshnessWindowSeconds: 3600,
|
|
41
|
+
freshnessFallback: "live",
|
|
42
|
+
};
|
|
43
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("serves the table at the exact window boundary (age == window)", () => {
|
|
47
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
48
|
+
tableName: "t",
|
|
49
|
+
dataAsOf: asOf(3600),
|
|
50
|
+
freshnessWindowSeconds: 3600,
|
|
51
|
+
freshnessFallback: "live",
|
|
52
|
+
};
|
|
53
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("drops a stale entry with fallback live (=> serve live)", () => {
|
|
57
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
58
|
+
tableName: "t",
|
|
59
|
+
dataAsOf: asOf(7200),
|
|
60
|
+
freshnessWindowSeconds: 3600,
|
|
61
|
+
freshnessFallback: "live",
|
|
62
|
+
};
|
|
63
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("keeps a stale entry with fallback stale_ok (=> serve the stale table)", () => {
|
|
67
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
68
|
+
tableName: "t",
|
|
69
|
+
dataAsOf: asOf(7200),
|
|
70
|
+
freshnessWindowSeconds: 3600,
|
|
71
|
+
freshnessFallback: "stale_ok",
|
|
72
|
+
};
|
|
73
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("drops a stale entry with fallback fail (interim: serve live in v1)", () => {
|
|
77
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
78
|
+
tableName: "t",
|
|
79
|
+
dataAsOf: asOf(7200),
|
|
80
|
+
freshnessWindowSeconds: 3600,
|
|
81
|
+
freshnessFallback: "fail",
|
|
82
|
+
};
|
|
83
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("drops a stale entry with no declared fallback (=> serve live)", () => {
|
|
87
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
88
|
+
tableName: "t",
|
|
89
|
+
dataAsOf: asOf(7200),
|
|
90
|
+
freshnessWindowSeconds: 3600,
|
|
91
|
+
};
|
|
92
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe("filterFreshManifest", () => {
|
|
97
|
+
it("includes fresh/un-gated/stale_ok and drops stale live/fail", () => {
|
|
98
|
+
const entries: FreshnessManifest = {
|
|
99
|
+
fresh: {
|
|
100
|
+
tableName: "fresh_t",
|
|
101
|
+
dataAsOf: asOf(1800),
|
|
102
|
+
freshnessWindowSeconds: 3600,
|
|
103
|
+
freshnessFallback: "live",
|
|
104
|
+
},
|
|
105
|
+
ungated: { tableName: "ungated_t", dataAsOf: asOf(999999) },
|
|
106
|
+
staleLive: {
|
|
107
|
+
tableName: "stale_live_t",
|
|
108
|
+
dataAsOf: asOf(7200),
|
|
109
|
+
freshnessWindowSeconds: 3600,
|
|
110
|
+
freshnessFallback: "live",
|
|
111
|
+
},
|
|
112
|
+
staleOk: {
|
|
113
|
+
tableName: "stale_ok_t",
|
|
114
|
+
dataAsOf: asOf(7200),
|
|
115
|
+
freshnessWindowSeconds: 3600,
|
|
116
|
+
freshnessFallback: "stale_ok",
|
|
117
|
+
},
|
|
118
|
+
staleFail: {
|
|
119
|
+
tableName: "stale_fail_t",
|
|
120
|
+
dataAsOf: asOf(7200),
|
|
121
|
+
freshnessWindowSeconds: 3600,
|
|
122
|
+
freshnessFallback: "fail",
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const { manifest } = filterFreshManifest(entries, NOW);
|
|
127
|
+
expect(Object.keys(manifest).sort()).toEqual([
|
|
128
|
+
"fresh",
|
|
129
|
+
"staleOk",
|
|
130
|
+
"ungated",
|
|
131
|
+
]);
|
|
132
|
+
expect(manifest.fresh).toEqual({ tableName: "fresh_t" });
|
|
133
|
+
expect(manifest.staleOk).toEqual({ tableName: "stale_ok_t" });
|
|
134
|
+
expect(manifest.ungated).toEqual({ tableName: "ungated_t" });
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("reports the earliest future window-crossing as nextStaleSince", () => {
|
|
138
|
+
const entries: FreshnessManifest = {
|
|
139
|
+
a: {
|
|
140
|
+
tableName: "a_t",
|
|
141
|
+
dataAsOf: asOf(1800), // window 3600 => flips at NOW + 1800s
|
|
142
|
+
freshnessWindowSeconds: 3600,
|
|
143
|
+
freshnessFallback: "live",
|
|
144
|
+
},
|
|
145
|
+
b: {
|
|
146
|
+
tableName: "b_t",
|
|
147
|
+
dataAsOf: asOf(1000), // window 3600 => flips at NOW + 2600s (later)
|
|
148
|
+
freshnessWindowSeconds: 3600,
|
|
149
|
+
freshnessFallback: "live",
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const { nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
154
|
+
// Earliest flip is entry a: dataAsOf(a) + window.
|
|
155
|
+
expect(nextStaleSince).toBe(NOW_MS - 1800 * 1000 + 3600 * 1000);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("kept stale_ok entries do not set a future nextStaleSince", () => {
|
|
159
|
+
const entries: FreshnessManifest = {
|
|
160
|
+
staleOk: {
|
|
161
|
+
tableName: "stale_ok_t",
|
|
162
|
+
dataAsOf: asOf(7200),
|
|
163
|
+
freshnessWindowSeconds: 3600,
|
|
164
|
+
freshnessFallback: "stale_ok",
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
168
|
+
expect(Object.keys(manifest)).toEqual(["staleOk"]);
|
|
169
|
+
// Its window crossing is already in the past, so nothing schedules a
|
|
170
|
+
// future recompute.
|
|
171
|
+
expect(nextStaleSince).toBeNull();
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("returns null nextStaleSince when nothing is gated", () => {
|
|
175
|
+
const entries: FreshnessManifest = {
|
|
176
|
+
a: { tableName: "a_t" },
|
|
177
|
+
b: { tableName: "b_t", dataAsOf: asOf(10) },
|
|
178
|
+
};
|
|
179
|
+
const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
180
|
+
expect(Object.keys(manifest).sort()).toEqual(["a", "b"]);
|
|
181
|
+
expect(nextStaleSince).toBeNull();
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-query freshness gate for persisted sources (persistence.md §9.3, §14
|
|
3
|
+
* Phase B). A query on a `#@ persist` source may use its materialized table
|
|
4
|
+
* only while that table is within its declared freshness window; otherwise the
|
|
5
|
+
* query does what the source's `fallback` says.
|
|
6
|
+
*
|
|
7
|
+
* Two settled design principles anchor this (do not re-derive them the easy
|
|
8
|
+
* way):
|
|
9
|
+
*
|
|
10
|
+
* 1. Enforce per query, not at bind. Filtering stale entries once when the
|
|
11
|
+
* manifest is bound would let a table that crosses its window while the
|
|
12
|
+
* package stays loaded keep serving stale. The serve path re-evaluates
|
|
13
|
+
* freshness on every query.
|
|
14
|
+
* 2. Clock on `dataAsOf`, not build completion. The window is a deadline on
|
|
15
|
+
* *data* age; the anchor is the artifact's data-as-of instant (snapshot
|
|
16
|
+
* start) supplied by the control plane. `age = now - dataAsOf`.
|
|
17
|
+
*
|
|
18
|
+
* Scope: `live`/`stale_ok` are fully publisher-side. `fail` degrades to
|
|
19
|
+
* serve-live in this first cut (the entry is dropped like `live`); the airtight
|
|
20
|
+
* `fail` path — which must *error* a query that joins a stale persist source —
|
|
21
|
+
* is a gated forked-malloy follow-up and is not built here.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
BuildManifest,
|
|
26
|
+
FreshnessAwareManifestEntry,
|
|
27
|
+
FreshnessManifest,
|
|
28
|
+
} from "../storage/DatabaseInterface";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* What to do with a single manifest entry at query time.
|
|
32
|
+
* - `serve_table`: substitute the materialized table (include `{ tableName }`).
|
|
33
|
+
* - `serve_live`: drop the entry so `strict:false` falls through to live SQL.
|
|
34
|
+
*
|
|
35
|
+
* `fail` collapses to `serve_live` in v1 (see module note).
|
|
36
|
+
*/
|
|
37
|
+
export type FreshnessDecision = "serve_table" | "serve_live";
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Evaluate one entry against `now`. An entry with no `freshnessWindowSeconds`
|
|
41
|
+
* is un-gated (always serve the table). A fresh entry (age within window) is
|
|
42
|
+
* served. A stale entry serves its table only under `stale_ok`; `live` and
|
|
43
|
+
* `fail` both drop to live in this first cut.
|
|
44
|
+
*/
|
|
45
|
+
export function evaluateManifestFreshness(
|
|
46
|
+
entry: FreshnessAwareManifestEntry,
|
|
47
|
+
now: Date,
|
|
48
|
+
): FreshnessDecision {
|
|
49
|
+
// Un-gated: no declared window ⇒ never stale. Also the rollout-safe default
|
|
50
|
+
// before the control plane starts stamping the freshness fields.
|
|
51
|
+
if (entry.freshnessWindowSeconds == null) return "serve_table";
|
|
52
|
+
// A window without an anchor can't be evaluated; treat as un-gated rather
|
|
53
|
+
// than tripping every query (matches "absent ⇒ never stale").
|
|
54
|
+
if (entry.dataAsOf == null) return "serve_table";
|
|
55
|
+
const dataAsOfMs = Date.parse(entry.dataAsOf);
|
|
56
|
+
if (Number.isNaN(dataAsOfMs)) return "serve_table";
|
|
57
|
+
const ageSeconds = (now.getTime() - dataAsOfMs) / 1000;
|
|
58
|
+
const stale = ageSeconds > entry.freshnessWindowSeconds;
|
|
59
|
+
if (!stale) return "serve_table";
|
|
60
|
+
return entry.freshnessFallback === "stale_ok" ? "serve_table" : "serve_live";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The instant a currently-fresh, gated entry will cross its window
|
|
65
|
+
* (`dataAsOf + window`), or `null` when the entry has no evaluable window
|
|
66
|
+
* (un-gated). Used to compute the next memo-invalidation instant.
|
|
67
|
+
*/
|
|
68
|
+
function staleSinceMs(entry: FreshnessAwareManifestEntry): number | null {
|
|
69
|
+
if (entry.freshnessWindowSeconds == null || entry.dataAsOf == null) {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
const dataAsOfMs = Date.parse(entry.dataAsOf);
|
|
73
|
+
if (Number.isNaN(dataAsOfMs)) return null;
|
|
74
|
+
return dataAsOfMs + entry.freshnessWindowSeconds * 1000;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Compute the freshness-filtered Malloy build manifest for `now` from the full
|
|
79
|
+
* retained wire entries.
|
|
80
|
+
*
|
|
81
|
+
* Returns the `sourceEntityId -> { tableName }` map of entries that should
|
|
82
|
+
* substitute their materialized table (fresh, un-gated, or stale-`stale_ok`),
|
|
83
|
+
* plus `nextStaleSince`: the earliest future instant a currently-included,
|
|
84
|
+
* gated entry will cross its window. Because age only ever grows, the
|
|
85
|
+
* fresh→stale transition is one-way, so a single `nextStaleSince` instant is a
|
|
86
|
+
* sufficient memo-invalidation deadline — no per-entry heap is needed. `null`
|
|
87
|
+
* means no included entry has an evaluable window, so the result never changes
|
|
88
|
+
* until the entries are replaced (rebind).
|
|
89
|
+
*/
|
|
90
|
+
export function filterFreshManifest(
|
|
91
|
+
entries: FreshnessManifest,
|
|
92
|
+
now: Date,
|
|
93
|
+
): { manifest: BuildManifest["entries"]; nextStaleSince: number | null } {
|
|
94
|
+
const manifest: BuildManifest["entries"] = {};
|
|
95
|
+
let nextStaleSince: number | null = null;
|
|
96
|
+
const nowMs = now.getTime();
|
|
97
|
+
|
|
98
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
99
|
+
if (evaluateManifestFreshness(entry, now) === "serve_live") continue;
|
|
100
|
+
manifest[sourceEntityId] = { tableName: entry.tableName };
|
|
101
|
+
|
|
102
|
+
// Only currently-fresh gated entries can flip later; a kept stale
|
|
103
|
+
// `stale_ok` entry (staleSince already in the past) never changes again.
|
|
104
|
+
const flipAt = staleSinceMs(entry);
|
|
105
|
+
if (flipAt != null && flipAt > nowMs) {
|
|
106
|
+
nextStaleSince =
|
|
107
|
+
nextStaleSince == null ? flipAt : Math.min(nextStaleSince, flipAt);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return { manifest, nextStaleSince };
|
|
112
|
+
}
|
|
@@ -46,6 +46,39 @@ describe("fetchManifestEntries", () => {
|
|
|
46
46
|
});
|
|
47
47
|
});
|
|
48
48
|
|
|
49
|
+
it("carries the control-plane freshness fields verbatim", async () => {
|
|
50
|
+
const dataAsOf = "2026-07-07T00:00:00.000Z";
|
|
51
|
+
const file = await writeManifest({
|
|
52
|
+
entries: {
|
|
53
|
+
gated: {
|
|
54
|
+
sourceEntityId: "gated",
|
|
55
|
+
physicalTableName: "schema.gated_mz",
|
|
56
|
+
dataAsOf,
|
|
57
|
+
freshnessWindowSeconds: 3600,
|
|
58
|
+
freshnessFallback: "stale_ok",
|
|
59
|
+
},
|
|
60
|
+
ungated: {
|
|
61
|
+
sourceEntityId: "ungated",
|
|
62
|
+
physicalTableName: "schema.ungated_mz",
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const entries = await fetchManifestEntries(file);
|
|
68
|
+
|
|
69
|
+
// Filter-free: freshness fields are retained for the serve-path gate; a
|
|
70
|
+
// window-less entry is left un-gated.
|
|
71
|
+
expect(entries).toEqual({
|
|
72
|
+
gated: {
|
|
73
|
+
tableName: "schema.gated_mz",
|
|
74
|
+
dataAsOf,
|
|
75
|
+
freshnessWindowSeconds: 3600,
|
|
76
|
+
freshnessFallback: "stale_ok",
|
|
77
|
+
},
|
|
78
|
+
ungated: { tableName: "schema.ungated_mz" },
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
49
82
|
it("reads via a file:// URI", async () => {
|
|
50
83
|
const file = await writeManifest({
|
|
51
84
|
entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
|