@malloy-publisher/server 0.0.223 → 0.0.225
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +75 -0
- package/dist/app/assets/{EnvironmentPage-3wCdljov.js → EnvironmentPage-DYTeXDll.js} +1 -1
- package/dist/app/assets/{HomePage-K0GHwqq2.js → HomePage-pDK2BPJY.js} +1 -1
- package/dist/app/assets/{LightMode-CwU3kN4I.js → LightMode-C2bwGPY1.js} +1 -1
- package/dist/app/assets/{MainPage-CTncHE5T.js → MainPage-WtBulMH_.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-DziAOD6-.js → MaterializationsPage-hMgOtflG.js} +1 -1
- package/dist/app/assets/{ModelPage-XrS2jXEc.js → ModelPage-B2N5kYII.js} +1 -1
- package/dist/app/assets/{PackagePage-BkwgFxG8.js → PackagePage-CEN90nQG.js} +1 -1
- package/dist/app/assets/{RouteError-9cIQga6p.js → RouteError-BG2c5Zf0.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DWiCRfEe.js → ThemeEditorPage-DNfeUwEZ.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Ce7FM_Po.js → WorkbookPage-NKI1BhFS.js} +1 -1
- package/dist/app/assets/{core-D9Hl0IDY.es-CrO01m4X.js → core-C6anj5c0.es-DDLHqpzt.js} +1 -1
- package/dist/app/assets/{index-D2sm5RBA.js → index-C6gZ6sSY.js} +5 -5
- package/dist/app/assets/{index-CtQm7kvp.js → index-CzNqKMVl.js} +1 -1
- package/dist/app/assets/{index-wJU_kzl2.js → index-DMQtnaf4.js} +2 -2
- package/dist/app/assets/{index-Dz8hgkmy.js → index-JXgvyZna.js} +1 -1
- package/dist/app/assets/{index-8E2uLeV9.js → index-OEjKNSYb.js} +2 -2
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +411 -26
- package/dist/sshcrypto-8m50vnmb.node +0 -0
- package/package.json +12 -12
- package/src/controller/materialization.controller.spec.ts +72 -0
- package/src/controller/materialization.controller.ts +75 -11
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/service/build_plan.spec.ts +119 -0
- package/src/service/build_plan.ts +143 -0
- package/src/service/environment.ts +3 -3
- package/src/service/freshness.spec.ts +183 -0
- package/src/service/freshness.ts +112 -0
- package/src/service/manifest_loader.spec.ts +33 -0
- package/src/service/manifest_loader.ts +17 -8
- package/src/service/materialization_cron_gate.spec.ts +61 -17
- package/src/service/materialization_service.spec.ts +42 -0
- package/src/service/materialization_service.ts +46 -3
- package/src/service/materialization_test_fixtures.ts +47 -14
- package/src/service/model.ts +54 -4
- package/src/service/package.ts +173 -35
- package/src/storage/DatabaseInterface.ts +24 -0
- package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
- package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
- package/tests/fixtures/persist-multi-level/publisher.json +5 -0
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +292 -0
- package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
package/src/service/package.ts
CHANGED
|
@@ -30,9 +30,14 @@ import {
|
|
|
30
30
|
import { formatDuration, logger } from "../logger";
|
|
31
31
|
import { recordBuildPlanComputeDuration } from "../materialization_metrics";
|
|
32
32
|
import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
|
|
33
|
-
import {
|
|
33
|
+
import {
|
|
34
|
+
BuildManifest,
|
|
35
|
+
BuildPlan,
|
|
36
|
+
FreshnessManifest,
|
|
37
|
+
} from "../storage/DatabaseInterface";
|
|
34
38
|
import { errMessage, ignoreDotfiles } from "../utils";
|
|
35
39
|
import { computePackageBuildPlan } from "./build_plan";
|
|
40
|
+
import { filterFreshManifest } from "./freshness";
|
|
36
41
|
import { Model } from "./model";
|
|
37
42
|
import { assertPersistNamesQuoted } from "./persist_annotation_validation";
|
|
38
43
|
|
|
@@ -52,6 +57,22 @@ type PackageConnectionInput =
|
|
|
52
57
|
| Map<string, Connection>
|
|
53
58
|
| (() => MalloyConfig);
|
|
54
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Project the full wire entries down to the Malloy-runtime binding map
|
|
62
|
+
* (`sourceEntityId -> { tableName }`) used to hydrate models. The freshness
|
|
63
|
+
* fields are dropped here — they gate the serve path per query, not model
|
|
64
|
+
* hydration.
|
|
65
|
+
*/
|
|
66
|
+
function toTableNameManifest(
|
|
67
|
+
entries: FreshnessManifest,
|
|
68
|
+
): BuildManifest["entries"] {
|
|
69
|
+
const out: BuildManifest["entries"] = {};
|
|
70
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
71
|
+
out[sourceEntityId] = { tableName: entry.tableName };
|
|
72
|
+
}
|
|
73
|
+
return out;
|
|
74
|
+
}
|
|
75
|
+
|
|
55
76
|
export class Package {
|
|
56
77
|
private environmentName: string;
|
|
57
78
|
private packageName: string;
|
|
@@ -67,6 +88,22 @@ export class Package {
|
|
|
67
88
|
// /status (via getPackageMetadata) so the control plane can confirm a worker
|
|
68
89
|
// actually bound the distributed manifest instead of inferring it from logs.
|
|
69
90
|
private buildManifestEntries: BuildManifest["entries"] | undefined;
|
|
91
|
+
// The full wire entries retained at bind (sourceEntityId -> { tableName,
|
|
92
|
+
// dataAsOf, freshnessWindowSeconds, freshnessFallback }). Unlike
|
|
93
|
+
// {@link buildManifestEntries} (the tableName-only projection baked into the
|
|
94
|
+
// hydration runtime and reused by /compile), this keeps the control-plane
|
|
95
|
+
// freshness fields so the serve path can re-evaluate `age vs window` per
|
|
96
|
+
// query. Undefined when the package is serving live (unbound).
|
|
97
|
+
private freshnessEntries: FreshnessManifest | undefined;
|
|
98
|
+
// Memoized freshness-filtered manifest for the serve path. Almost all queries
|
|
99
|
+
// in a window share the same included set, so this is recomputed only when a
|
|
100
|
+
// retained entry actually crosses its window (`validUntil`, the next
|
|
101
|
+
// staleSince) or when a rebind replaces the entries (cleared in
|
|
102
|
+
// recordManifestBinding). validUntil === Infinity means no included entry has
|
|
103
|
+
// an evaluable window, so the result is stable until the next rebind.
|
|
104
|
+
private freshManifestCache:
|
|
105
|
+
| { manifest: BuildManifest["entries"]; validUntil: number }
|
|
106
|
+
| undefined;
|
|
70
107
|
private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
|
|
71
108
|
"unbound";
|
|
72
109
|
private manifestEntryCount = 0;
|
|
@@ -419,6 +456,10 @@ export class Package {
|
|
|
419
456
|
malloyConfig,
|
|
420
457
|
);
|
|
421
458
|
pkg.renderTagWarnings = renderTagWarnings;
|
|
459
|
+
// Install the per-query freshness resolver on the freshly-built models.
|
|
460
|
+
// At create time no manifest is bound yet, so the resolver returns
|
|
461
|
+
// undefined (serve live) until a subsequent bindManifest → reloadAllModels.
|
|
462
|
+
pkg.wireFreshnessResolvers();
|
|
422
463
|
|
|
423
464
|
// Compute the persist build plan off the live (unbound) models, before the
|
|
424
465
|
// caller binds any configured manifest, so the surfaced plan reflects the
|
|
@@ -480,6 +521,17 @@ export class Package {
|
|
|
480
521
|
return this.buildPlan;
|
|
481
522
|
}
|
|
482
523
|
|
|
524
|
+
/**
|
|
525
|
+
* The package-level `materialization` config (from malloy-publisher.json),
|
|
526
|
+
* the least-specific layer for resolving per-source freshness/schedule in the
|
|
527
|
+
* build plan. Null when the package declares no policy.
|
|
528
|
+
*/
|
|
529
|
+
public getMaterializationConfig(): NonNullable<
|
|
530
|
+
ApiPackage["materialization"]
|
|
531
|
+
> | null {
|
|
532
|
+
return this.packageMetadata.materialization ?? null;
|
|
533
|
+
}
|
|
534
|
+
|
|
483
535
|
public getPackageMetadata(): ApiPackage {
|
|
484
536
|
// Overlay the server-computed fields onto the stored metadata: the
|
|
485
537
|
// explores misconfig warnings (loading is fail-safe — the package still
|
|
@@ -523,6 +575,51 @@ export class Package {
|
|
|
523
575
|
return this.buildManifestEntries;
|
|
524
576
|
}
|
|
525
577
|
|
|
578
|
+
/**
|
|
579
|
+
* The freshness-filtered build manifest for the serve path, evaluated at
|
|
580
|
+
* `now`. Persistence.md §9.3 Phase B: a query on a `#@ persist` source may
|
|
581
|
+
* use its materialized table only while the table is within its declared
|
|
582
|
+
* freshness window; otherwise it falls back per the entry's `fallback`
|
|
583
|
+
* (`live`/`fail` drop the entry → serve live; `stale_ok` keeps it → serve the
|
|
584
|
+
* stale table). Un-gated entries (no window) always route to the table.
|
|
585
|
+
*
|
|
586
|
+
* Undefined when the package is serving live (unbound) — callers then apply
|
|
587
|
+
* no per-query override and the runtime serves live.
|
|
588
|
+
*
|
|
589
|
+
* Memoized: recomputed only when a retained entry crosses its window
|
|
590
|
+
* (monotonic fresh→stale, so a single `validUntil` deadline suffices) or when
|
|
591
|
+
* a rebind replaces the entries. Cheap O(1) hit on the common path.
|
|
592
|
+
*/
|
|
593
|
+
public getFreshBuildManifest(
|
|
594
|
+
now: number = Date.now(),
|
|
595
|
+
): BuildManifest["entries"] | undefined {
|
|
596
|
+
if (!this.freshnessEntries) return undefined;
|
|
597
|
+
if (this.freshManifestCache && now < this.freshManifestCache.validUntil) {
|
|
598
|
+
return this.freshManifestCache.manifest;
|
|
599
|
+
}
|
|
600
|
+
const { manifest, nextStaleSince } = filterFreshManifest(
|
|
601
|
+
this.freshnessEntries,
|
|
602
|
+
new Date(now),
|
|
603
|
+
);
|
|
604
|
+
this.freshManifestCache = {
|
|
605
|
+
manifest,
|
|
606
|
+
validUntil: nextStaleSince ?? Infinity,
|
|
607
|
+
};
|
|
608
|
+
return manifest;
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
/**
|
|
612
|
+
* Install the per-query freshness resolver on every owned model so the serve
|
|
613
|
+
* path (Model.getQueryResults / executeNotebookCell) applies the
|
|
614
|
+
* freshness-filtered manifest as a per-query Malloy `buildManifest` override.
|
|
615
|
+
* Idempotent; called after (re)building the model set.
|
|
616
|
+
*/
|
|
617
|
+
private wireFreshnessResolvers(): void {
|
|
618
|
+
for (const model of this.models.values()) {
|
|
619
|
+
model.setFreshnessResolver(() => this.getFreshBuildManifest());
|
|
620
|
+
}
|
|
621
|
+
}
|
|
622
|
+
|
|
526
623
|
/**
|
|
527
624
|
* Record the URI whose manifest is currently bound to the served models. May
|
|
528
625
|
* differ from `manifestLocation` after an in-memory auto-load following a
|
|
@@ -545,13 +642,18 @@ export class Package {
|
|
|
545
642
|
* observability (/status). An empty map means the manifest was cleared, so
|
|
546
643
|
* the package reverts to live (unbound).
|
|
547
644
|
*/
|
|
548
|
-
private recordManifestBinding(
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
this.
|
|
645
|
+
private recordManifestBinding(entries: FreshnessManifest): void {
|
|
646
|
+
const count = Object.keys(entries).length;
|
|
647
|
+
// Full wire entries (with freshness) drive the per-query serve-path gate;
|
|
648
|
+
// the tableName-only projection is what /compile and /status consume.
|
|
649
|
+
this.freshnessEntries = count > 0 ? entries : undefined;
|
|
650
|
+
this.buildManifestEntries =
|
|
651
|
+
count > 0 ? toTableNameManifest(entries) : undefined;
|
|
553
652
|
this.manifestEntryCount = count;
|
|
554
653
|
this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
|
|
654
|
+
// A rebind replaces the entries, so any memoized freshness-filtered
|
|
655
|
+
// manifest is stale — drop it so the next query recomputes.
|
|
656
|
+
this.freshManifestCache = undefined;
|
|
555
657
|
if (count === 0) {
|
|
556
658
|
this.boundManifestUri = null;
|
|
557
659
|
}
|
|
@@ -615,33 +717,61 @@ export class Package {
|
|
|
615
717
|
}
|
|
616
718
|
|
|
617
719
|
/**
|
|
618
|
-
* Publish-gate for
|
|
619
|
-
* of artifact-anchored scheduling
|
|
620
|
-
*
|
|
621
|
-
*
|
|
622
|
-
*
|
|
623
|
-
*
|
|
624
|
-
*
|
|
625
|
-
*
|
|
626
|
-
*
|
|
627
|
-
*
|
|
628
|
-
*
|
|
720
|
+
* Publish-gate for materialization crons (Phase A carve-out, persistence.md
|
|
721
|
+
* §9.4). A cron is the power tier of artifact-anchored scheduling and is
|
|
722
|
+
* valid only over `sharing="private"` artifacts; a cron on a shared (or unset
|
|
723
|
+
* ⇒ shared) artifact is rejected, pointing at `freshness.window` as the
|
|
724
|
+
* shared-tier replacement. Two grains, both enforced off the resolved build
|
|
725
|
+
* plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
|
|
726
|
+
*
|
|
727
|
+
* - Per-source: a source's own `schedule` is valid only when that source
|
|
728
|
+
* resolves to explicit `sharing="private"`.
|
|
729
|
+
* - Package: `materialization.schedule` governs every persist source, so it
|
|
730
|
+
* is valid only when every governed source resolves to explicit
|
|
731
|
+
* `sharing="private"` (a mixed/shared/unset package cannot carry it).
|
|
732
|
+
*
|
|
733
|
+
* Strict at publish (package.controller), warn-only at load/reload
|
|
734
|
+
* (loadViaWorker) — same split as explores.
|
|
629
735
|
*/
|
|
630
736
|
public scheduleWarnings(): string[] {
|
|
631
|
-
const
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
737
|
+
const warnings: string[] = [];
|
|
738
|
+
const sources = this.buildPlan?.sources
|
|
739
|
+
? Object.values(this.buildPlan.sources)
|
|
740
|
+
: [];
|
|
741
|
+
|
|
742
|
+
for (const source of sources) {
|
|
743
|
+
if (source.schedule && source.sharing !== "private") {
|
|
744
|
+
warnings.push(
|
|
745
|
+
`#@ persist source "${source.name}" declares a schedule (cron) but ` +
|
|
746
|
+
`resolves to sharing="${
|
|
747
|
+
source.sharing ?? "shared (unset default)"
|
|
748
|
+
}": a per-source cron is valid only for sharing="private". ` +
|
|
749
|
+
`Declare 'freshness.window' instead (the control plane schedules ` +
|
|
750
|
+
`refreshes from it).`,
|
|
751
|
+
);
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
const packageSchedule = this.packageMetadata.materialization?.schedule;
|
|
756
|
+
if (packageSchedule) {
|
|
757
|
+
const nonPrivate = sources.filter((s) => s.sharing !== "private");
|
|
758
|
+
if (sources.length === 0 || nonPrivate.length > 0) {
|
|
759
|
+
const detail =
|
|
760
|
+
sources.length === 0
|
|
761
|
+
? "the package declares no persist source for it to govern"
|
|
762
|
+
: `sources [${nonPrivate
|
|
763
|
+
.map((s) => s.name)
|
|
764
|
+
.join(", ")}] resolve to shared/unset`;
|
|
765
|
+
warnings.push(
|
|
766
|
+
`materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` +
|
|
767
|
+
`only when every persist source resolves to sharing="private": ` +
|
|
768
|
+
`${detail}. Declare 'materialization.freshness.window' instead ` +
|
|
769
|
+
`(the control plane schedules refreshes from it).`,
|
|
770
|
+
);
|
|
771
|
+
}
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
return warnings;
|
|
645
775
|
}
|
|
646
776
|
|
|
647
777
|
/** The {@link scheduleWarnings} joined into one string, or "" if none. */
|
|
@@ -728,9 +858,11 @@ export class Package {
|
|
|
728
858
|
* timeout, pool shutting down) propagate as `ServiceUnavailableError`
|
|
729
859
|
* — the caller (manifest service) decides how to retry.
|
|
730
860
|
*/
|
|
731
|
-
public async reloadAllModels(
|
|
732
|
-
|
|
733
|
-
|
|
861
|
+
public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
|
|
862
|
+
// Models are hydrated against the tableName-only projection; the freshness
|
|
863
|
+
// fields gate the serve path per query (via getFreshBuildManifest), not
|
|
864
|
+
// model hydration.
|
|
865
|
+
const buildManifest = toTableNameManifest(entries);
|
|
734
866
|
const modelPaths = Array.from(this.models.keys());
|
|
735
867
|
logger.info("Reloading all models with build manifest", {
|
|
736
868
|
packageName: this.packageName,
|
|
@@ -837,7 +969,13 @@ export class Package {
|
|
|
837
969
|
this.applyQueryBoundaryToModels();
|
|
838
970
|
// Remember what we just bound so /compile can route identically and
|
|
839
971
|
// /status can report the binding. An empty map reverts to live (unbound).
|
|
840
|
-
|
|
972
|
+
// Retains the full freshness entries so the serve-path gate can evaluate
|
|
973
|
+
// them per query.
|
|
974
|
+
this.recordManifestBinding(entries);
|
|
975
|
+
// Install the per-query freshness resolver on the rebuilt model set so the
|
|
976
|
+
// serve path applies the freshness-filtered manifest as a per-query
|
|
977
|
+
// override.
|
|
978
|
+
this.wireFreshnessResolvers();
|
|
841
979
|
// Re-run the fail-safe warning against the refreshed model set: an edit
|
|
842
980
|
// to publisher.json that introduces a bad entry should surface in the
|
|
843
981
|
// logs on reload too, not only at initial load (loadViaWorker).
|
|
@@ -116,6 +116,7 @@ export type BuildPlan = components["schemas"]["BuildPlan"];
|
|
|
116
116
|
export type BuildManifestResult = components["schemas"]["BuildManifest"];
|
|
117
117
|
export type ManifestEntry = components["schemas"]["ManifestEntry"];
|
|
118
118
|
export type BuildInstruction = components["schemas"]["BuildInstruction"];
|
|
119
|
+
export type ManifestReference = components["schemas"]["ManifestReference"];
|
|
119
120
|
export type Realization = components["schemas"]["Realization"];
|
|
120
121
|
|
|
121
122
|
export interface Materialization {
|
|
@@ -157,3 +158,26 @@ export interface BuildManifest {
|
|
|
157
158
|
entries: Record<string, BuildManifestEntry>;
|
|
158
159
|
strict?: boolean;
|
|
159
160
|
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* A wire manifest entry carrying the control-plane freshness fields alongside
|
|
164
|
+
* the physical `tableName`. This is what the publisher retains at bind so the
|
|
165
|
+
* serve path can re-evaluate `age vs window` per query (see
|
|
166
|
+
* {@link ../service/freshness}). A `{ tableName }`-only value is structurally
|
|
167
|
+
* assignable here (the freshness fields are optional), so callers that only
|
|
168
|
+
* know a table name — e.g. the post-build auto-load — bind un-gated entries.
|
|
169
|
+
*
|
|
170
|
+
* - `dataAsOf` the artifact's data-as-of instant (snapshot start); the age
|
|
171
|
+
* anchor. `age = now - dataAsOf`. Absent ⇒ never stale.
|
|
172
|
+
* - `freshnessWindowSeconds` the version's effective window. Absent ⇒ un-gated.
|
|
173
|
+
* - `freshnessFallback` per-version read-time policy when stale.
|
|
174
|
+
*/
|
|
175
|
+
export interface FreshnessAwareManifestEntry {
|
|
176
|
+
tableName: string;
|
|
177
|
+
dataAsOf?: string;
|
|
178
|
+
freshnessWindowSeconds?: number;
|
|
179
|
+
freshnessFallback?: "live" | "stale_ok" | "fail";
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Full (unfiltered) manifest map keyed by sourceEntityId. */
|
|
183
|
+
export type FreshnessManifest = Record<string, FreshnessAwareManifestEntry>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
##! experimental.persistence
|
|
2
|
+
|
|
3
|
+
source: raw_orders is duckdb.table('data/orders.csv')
|
|
4
|
+
|
|
5
|
+
// Upstream persist source: an aggregate over the base CSV.
|
|
6
|
+
#@ persist name="orders_base"
|
|
7
|
+
source: orders_base is raw_orders -> {
|
|
8
|
+
group_by: category
|
|
9
|
+
aggregate: total_amount is amount.sum()
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// Downstream persist source: references the persist upstream `orders_base`.
|
|
13
|
+
// A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
|
|
14
|
+
// materialized table (via referenceManifest) rather than recompute it live.
|
|
15
|
+
#@ persist name="orders_rollup"
|
|
16
|
+
source: orders_rollup is orders_base -> {
|
|
17
|
+
aggregate: category_count is count()
|
|
18
|
+
}
|
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
/// <reference types="bun-types" />
|
|
2
|
+
|
|
3
|
+
import { afterAll, beforeAll, describe, expect, it } from "bun:test";
|
|
4
|
+
import * as fsp from "fs/promises";
|
|
5
|
+
import os from "os";
|
|
6
|
+
import path from "path";
|
|
7
|
+
import { fileURLToPath } from "url";
|
|
8
|
+
import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
|
|
9
|
+
|
|
10
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
11
|
+
const __dirname = path.dirname(__filename);
|
|
12
|
+
|
|
13
|
+
const PROJECT_NAME = "freshness-gate-project";
|
|
14
|
+
const PACKAGE_NAME = "persist-test";
|
|
15
|
+
const MODEL_PATH = "persist_test.malloy";
|
|
16
|
+
const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Covers the per-query freshness gate (persistence.md §9.3, §14 Phase B): a
|
|
20
|
+
* query on a `#@ persist` source uses its materialized table only while the
|
|
21
|
+
* table is within its declared freshness window; otherwise it falls back per the
|
|
22
|
+
* entry's declared `fallback`.
|
|
23
|
+
*
|
|
24
|
+
* - fresh (or un-gated) → routes to the materialized table
|
|
25
|
+
* - stale + fallback live → serves live SQL (entry dropped)
|
|
26
|
+
* - stale + fallback stale_ok→ serves the stale table
|
|
27
|
+
* - a fresh entry that crosses its window → serves live on the NEXT query,
|
|
28
|
+
* with no rebind (proves per-query re-evaluation, not bind-time filtering)
|
|
29
|
+
*
|
|
30
|
+
* The manifest is bound once physically (auto-run builds the table), then a
|
|
31
|
+
* manifest URI keyed by the source's real sourceEntityId routes served queries.
|
|
32
|
+
* `executedSql()` is the routing evidence: it scans `order_summary` when routed
|
|
33
|
+
* to the table, and `data/orders.csv` when serving live.
|
|
34
|
+
*/
|
|
35
|
+
describe("Per-query freshness gate (E2E)", () => {
|
|
36
|
+
let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
|
|
37
|
+
let baseUrl: string;
|
|
38
|
+
let tmpDir: string;
|
|
39
|
+
let sourceEntityId: string;
|
|
40
|
+
|
|
41
|
+
beforeAll(async () => {
|
|
42
|
+
env = await startRestE2E();
|
|
43
|
+
baseUrl = env.baseUrl;
|
|
44
|
+
tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "freshness-gate-"));
|
|
45
|
+
|
|
46
|
+
const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
|
|
47
|
+
const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
|
|
48
|
+
method: "POST",
|
|
49
|
+
headers: { "Content-Type": "application/json" },
|
|
50
|
+
body: JSON.stringify({
|
|
51
|
+
name: PROJECT_NAME,
|
|
52
|
+
packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
|
|
53
|
+
connections: [],
|
|
54
|
+
}),
|
|
55
|
+
});
|
|
56
|
+
if (!createRes.ok) {
|
|
57
|
+
throw new Error(
|
|
58
|
+
`Failed to create test project (${createRes.status}): ${await createRes.text()}`,
|
|
59
|
+
);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
const deadline = Date.now() + 30_000;
|
|
63
|
+
while (Date.now() < deadline) {
|
|
64
|
+
const res = await fetch(`${baseUrl}${API}`);
|
|
65
|
+
if (res.ok) break;
|
|
66
|
+
await new Promise((r) => setTimeout(r, 500));
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
// Physically build the persist source (auto-run self-assigns
|
|
70
|
+
// physicalTableName = "order_summary"), then revert to live so the
|
|
71
|
+
// manifest-URI bind is the only thing that can route the served query.
|
|
72
|
+
await buildTableThenRevertToLive();
|
|
73
|
+
sourceEntityId = await orderSummarySourceEntityId();
|
|
74
|
+
}, 120_000);
|
|
75
|
+
|
|
76
|
+
afterAll(async () => {
|
|
77
|
+
if (baseUrl) {
|
|
78
|
+
await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
|
|
79
|
+
method: "DELETE",
|
|
80
|
+
}).catch(() => {});
|
|
81
|
+
}
|
|
82
|
+
await env?.stop();
|
|
83
|
+
await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
|
|
84
|
+
env = null;
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
function url(p: string): string {
|
|
88
|
+
return `${baseUrl}${API}${p}`;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
const ROUTING_QUERY = "run: order_summary -> { aggregate: c is count() }";
|
|
92
|
+
|
|
93
|
+
/** The SQL the served query actually compiled to (routing evidence). */
|
|
94
|
+
async function executedSql(): Promise<string> {
|
|
95
|
+
const res = await fetch(`${baseUrl}${API}/models/${MODEL_PATH}/query`, {
|
|
96
|
+
method: "POST",
|
|
97
|
+
headers: { "Content-Type": "application/json" },
|
|
98
|
+
body: JSON.stringify({ query: ROUTING_QUERY }),
|
|
99
|
+
});
|
|
100
|
+
expect(res.status).toBe(200);
|
|
101
|
+
const body = (await res.json()) as { result: string };
|
|
102
|
+
return (JSON.parse(body.result) as { sql: string }).sql;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function orderSummarySourceEntityId(): Promise<string> {
|
|
106
|
+
const res = await fetch(url(""));
|
|
107
|
+
expect(res.status).toBe(200);
|
|
108
|
+
const pkg = (await res.json()) as {
|
|
109
|
+
buildPlan?: { sources?: Record<string, { sourceEntityId?: string }> };
|
|
110
|
+
};
|
|
111
|
+
const sources = pkg.buildPlan?.sources ?? {};
|
|
112
|
+
const id = Object.values(sources)[0]?.sourceEntityId;
|
|
113
|
+
expect(typeof id).toBe("string");
|
|
114
|
+
return id as string;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function pollUntilTerminal(
|
|
118
|
+
id: string,
|
|
119
|
+
timeoutMs = 90_000,
|
|
120
|
+
): Promise<Record<string, unknown>> {
|
|
121
|
+
const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
|
|
122
|
+
const deadline = Date.now() + timeoutMs;
|
|
123
|
+
while (Date.now() < deadline) {
|
|
124
|
+
const res = await fetch(url(`/materializations/${id}`));
|
|
125
|
+
expect(res.status).toBe(200);
|
|
126
|
+
const data = (await res.json()) as Record<string, unknown>;
|
|
127
|
+
if (terminal.includes(data.status as string)) return data;
|
|
128
|
+
await new Promise((r) => setTimeout(r, 250));
|
|
129
|
+
}
|
|
130
|
+
throw new Error(`Materialization ${id} did not reach a terminal state`);
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
async function buildTableThenRevertToLive(): Promise<void> {
|
|
134
|
+
const createRes = await fetch(url("/materializations"), {
|
|
135
|
+
method: "POST",
|
|
136
|
+
headers: { "Content-Type": "application/json" },
|
|
137
|
+
body: JSON.stringify({}),
|
|
138
|
+
});
|
|
139
|
+
expect(createRes.status).toBe(201);
|
|
140
|
+
const { id } = (await createRes.json()) as { id: string };
|
|
141
|
+
const built = await pollUntilTerminal(id);
|
|
142
|
+
expect(built.status).toBe("MANIFEST_FILE_READY");
|
|
143
|
+
await fetch(url("?reload=true"));
|
|
144
|
+
await fetch(url(`/materializations/${id}`), { method: "DELETE" });
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/** Write a CP-shaped manifest carrying the freshness fields. */
|
|
148
|
+
async function writeFreshnessManifest(opts: {
|
|
149
|
+
dataAsOf: string;
|
|
150
|
+
freshnessWindowSeconds: number;
|
|
151
|
+
freshnessFallback: "live" | "stale_ok" | "fail";
|
|
152
|
+
}): Promise<string> {
|
|
153
|
+
const file = path.join(
|
|
154
|
+
tmpDir,
|
|
155
|
+
`freshness-${Date.now()}-${Math.random()}.json`,
|
|
156
|
+
);
|
|
157
|
+
await fsp.writeFile(
|
|
158
|
+
file,
|
|
159
|
+
JSON.stringify({
|
|
160
|
+
builtAt: new Date().toISOString(),
|
|
161
|
+
strict: false,
|
|
162
|
+
entries: {
|
|
163
|
+
[sourceEntityId]: {
|
|
164
|
+
sourceEntityId,
|
|
165
|
+
sourceName: "order_summary",
|
|
166
|
+
physicalTableName: "order_summary",
|
|
167
|
+
connectionName: "duckdb",
|
|
168
|
+
dataAsOf: opts.dataAsOf,
|
|
169
|
+
freshnessWindowSeconds: opts.freshnessWindowSeconds,
|
|
170
|
+
freshnessFallback: opts.freshnessFallback,
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
}),
|
|
174
|
+
"utf8",
|
|
175
|
+
);
|
|
176
|
+
return file;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async function bind(manifestFile: string): Promise<void> {
|
|
180
|
+
const patchRes = await fetch(url(""), {
|
|
181
|
+
method: "PATCH",
|
|
182
|
+
headers: { "Content-Type": "application/json" },
|
|
183
|
+
body: JSON.stringify({
|
|
184
|
+
name: PACKAGE_NAME,
|
|
185
|
+
manifestLocation: manifestFile,
|
|
186
|
+
}),
|
|
187
|
+
});
|
|
188
|
+
expect(patchRes.status).toBe(200);
|
|
189
|
+
const patched = (await patchRes.json()) as Record<string, unknown>;
|
|
190
|
+
// The bind is recorded regardless of per-query freshness — freshness is
|
|
191
|
+
// evaluated per query, not at bind.
|
|
192
|
+
expect(patched.manifestBindingStatus).toBe("bound");
|
|
193
|
+
expect(patched.manifestEntryCount).toBe(1);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
async function revertToLive(): Promise<void> {
|
|
197
|
+
await fetch(url(""), {
|
|
198
|
+
method: "PATCH",
|
|
199
|
+
headers: { "Content-Type": "application/json" },
|
|
200
|
+
body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
|
|
201
|
+
});
|
|
202
|
+
await fetch(url("?reload=true"));
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
it(
|
|
206
|
+
"routes a fresh entry to the materialized table",
|
|
207
|
+
async () => {
|
|
208
|
+
const manifestFile = await writeFreshnessManifest({
|
|
209
|
+
dataAsOf: new Date().toISOString(),
|
|
210
|
+
freshnessWindowSeconds: 86_400, // 1 day — comfortably fresh
|
|
211
|
+
freshnessFallback: "live",
|
|
212
|
+
});
|
|
213
|
+
await bind(manifestFile);
|
|
214
|
+
|
|
215
|
+
const routed = await executedSql();
|
|
216
|
+
expect(routed).not.toContain("data/orders.csv");
|
|
217
|
+
expect(routed).toContain("order_summary");
|
|
218
|
+
|
|
219
|
+
await revertToLive();
|
|
220
|
+
},
|
|
221
|
+
{ timeout: 120_000 },
|
|
222
|
+
);
|
|
223
|
+
|
|
224
|
+
it(
|
|
225
|
+
"serves live for a stale entry with fallback live",
|
|
226
|
+
async () => {
|
|
227
|
+
const manifestFile = await writeFreshnessManifest({
|
|
228
|
+
// 2h old against a 1h window ⇒ stale.
|
|
229
|
+
dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
|
|
230
|
+
freshnessWindowSeconds: 3600,
|
|
231
|
+
freshnessFallback: "live",
|
|
232
|
+
});
|
|
233
|
+
await bind(manifestFile);
|
|
234
|
+
|
|
235
|
+
const served = await executedSql();
|
|
236
|
+
expect(served).toContain("data/orders.csv");
|
|
237
|
+
|
|
238
|
+
await revertToLive();
|
|
239
|
+
},
|
|
240
|
+
{ timeout: 120_000 },
|
|
241
|
+
);
|
|
242
|
+
|
|
243
|
+
it(
|
|
244
|
+
"serves the stale table for a stale entry with fallback stale_ok",
|
|
245
|
+
async () => {
|
|
246
|
+
const manifestFile = await writeFreshnessManifest({
|
|
247
|
+
dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
|
|
248
|
+
freshnessWindowSeconds: 3600,
|
|
249
|
+
freshnessFallback: "stale_ok",
|
|
250
|
+
});
|
|
251
|
+
await bind(manifestFile);
|
|
252
|
+
|
|
253
|
+
const served = await executedSql();
|
|
254
|
+
expect(served).not.toContain("data/orders.csv");
|
|
255
|
+
expect(served).toContain("order_summary");
|
|
256
|
+
|
|
257
|
+
await revertToLive();
|
|
258
|
+
},
|
|
259
|
+
{ timeout: 120_000 },
|
|
260
|
+
);
|
|
261
|
+
|
|
262
|
+
it(
|
|
263
|
+
"serves live once a fresh entry crosses its window, with no rebind",
|
|
264
|
+
async () => {
|
|
265
|
+
const windowSeconds = 10;
|
|
266
|
+
const anchor = Date.now();
|
|
267
|
+
const manifestFile = await writeFreshnessManifest({
|
|
268
|
+
dataAsOf: new Date(anchor).toISOString(),
|
|
269
|
+
freshnessWindowSeconds: windowSeconds,
|
|
270
|
+
freshnessFallback: "live",
|
|
271
|
+
});
|
|
272
|
+
await bind(manifestFile);
|
|
273
|
+
|
|
274
|
+
// While inside the window, the served query routes to the table.
|
|
275
|
+
expect(Date.now() - anchor).toBeLessThan(windowSeconds * 1000);
|
|
276
|
+
const fresh = await executedSql();
|
|
277
|
+
expect(fresh).not.toContain("data/orders.csv");
|
|
278
|
+
expect(fresh).toContain("order_summary");
|
|
279
|
+
|
|
280
|
+
// Wait for the window to elapse — no rebind, no reload.
|
|
281
|
+
const waitMs = windowSeconds * 1000 - (Date.now() - anchor) + 1500;
|
|
282
|
+
await new Promise((r) => setTimeout(r, Math.max(waitMs, 0)));
|
|
283
|
+
|
|
284
|
+
// The very next query re-evaluates freshness and now serves live.
|
|
285
|
+
const afterCrossing = await executedSql();
|
|
286
|
+
expect(afterCrossing).toContain("data/orders.csv");
|
|
287
|
+
|
|
288
|
+
await revertToLive();
|
|
289
|
+
},
|
|
290
|
+
{ timeout: 120_000 },
|
|
291
|
+
);
|
|
292
|
+
});
|