@malloy-publisher/server 0.0.223 → 0.0.224
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/app/api-doc.yaml +17 -0
- package/dist/app/assets/{EnvironmentPage-3wCdljov.js → EnvironmentPage-DYTeXDll.js} +1 -1
- package/dist/app/assets/{HomePage-K0GHwqq2.js → HomePage-pDK2BPJY.js} +1 -1
- package/dist/app/assets/{LightMode-CwU3kN4I.js → LightMode-C2bwGPY1.js} +1 -1
- package/dist/app/assets/{MainPage-CTncHE5T.js → MainPage-WtBulMH_.js} +2 -2
- package/dist/app/assets/{MaterializationsPage-DziAOD6-.js → MaterializationsPage-hMgOtflG.js} +1 -1
- package/dist/app/assets/{ModelPage-XrS2jXEc.js → ModelPage-B2N5kYII.js} +1 -1
- package/dist/app/assets/{PackagePage-BkwgFxG8.js → PackagePage-CEN90nQG.js} +1 -1
- package/dist/app/assets/{RouteError-9cIQga6p.js → RouteError-BG2c5Zf0.js} +1 -1
- package/dist/app/assets/{ThemeEditorPage-DWiCRfEe.js → ThemeEditorPage-DNfeUwEZ.js} +1 -1
- package/dist/app/assets/{WorkbookPage-Ce7FM_Po.js → WorkbookPage-NKI1BhFS.js} +1 -1
- package/dist/app/assets/{core-D9Hl0IDY.es-CrO01m4X.js → core-C6anj5c0.es-DDLHqpzt.js} +1 -1
- package/dist/app/assets/{index-D2sm5RBA.js → index-C6gZ6sSY.js} +5 -5
- package/dist/app/assets/{index-CtQm7kvp.js → index-CzNqKMVl.js} +1 -1
- package/dist/app/assets/{index-wJU_kzl2.js → index-DMQtnaf4.js} +2 -2
- package/dist/app/assets/{index-Dz8hgkmy.js → index-JXgvyZna.js} +1 -1
- package/dist/app/assets/{index-8E2uLeV9.js → index-OEjKNSYb.js} +2 -2
- package/dist/app/index.html +1 -1
- package/dist/server.mjs +266 -13
- package/package.json +12 -12
- package/src/mcp/skills/skills_bundle.json +1 -1
- package/src/service/environment.ts +3 -3
- package/src/service/freshness.spec.ts +183 -0
- package/src/service/freshness.ts +112 -0
- package/src/service/manifest_loader.spec.ts +33 -0
- package/src/service/manifest_loader.ts +17 -8
- package/src/service/materialization_cron_gate.spec.ts +23 -18
- package/src/service/materialization_service.ts +6 -1
- package/src/service/model.ts +54 -4
- package/src/service/package.ts +126 -33
- package/src/storage/DatabaseInterface.ts +23 -0
- package/tests/integration/materialization/freshness_gate.integration.spec.ts +292 -0
|
@@ -28,7 +28,7 @@ import {
|
|
|
28
28
|
assertSafeRelativeModelPath,
|
|
29
29
|
safeJoinUnderRoot,
|
|
30
30
|
} from "../path_safety";
|
|
31
|
-
import {
|
|
31
|
+
import { FreshnessManifest } from "../storage/DatabaseInterface";
|
|
32
32
|
import { URL_READER } from "../utils";
|
|
33
33
|
import {
|
|
34
34
|
buildEnvironmentMalloyConfig,
|
|
@@ -1119,7 +1119,7 @@ export class Environment {
|
|
|
1119
1119
|
*/
|
|
1120
1120
|
public async reloadAllModelsForPackage(
|
|
1121
1121
|
packageName: string,
|
|
1122
|
-
manifest:
|
|
1122
|
+
manifest: FreshnessManifest,
|
|
1123
1123
|
): Promise<void> {
|
|
1124
1124
|
assertSafePackageName(packageName);
|
|
1125
1125
|
return this.withPackageLock(packageName, async () => {
|
|
@@ -1193,7 +1193,7 @@ export class Environment {
|
|
|
1193
1193
|
*/
|
|
1194
1194
|
private async fetchManifestEntriesWithTimeout(
|
|
1195
1195
|
manifestLocation: string,
|
|
1196
|
-
): Promise<
|
|
1196
|
+
): Promise<FreshnessManifest> {
|
|
1197
1197
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
1198
1198
|
const timeout = new Promise<never>((_, reject) => {
|
|
1199
1199
|
timer = setTimeout(
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
import { describe, expect, it } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import type {
|
|
4
|
+
FreshnessAwareManifestEntry,
|
|
5
|
+
FreshnessManifest,
|
|
6
|
+
} from "../storage/DatabaseInterface";
|
|
7
|
+
import { evaluateManifestFreshness, filterFreshManifest } from "./freshness";
|
|
8
|
+
|
|
9
|
+
// A fixed "now" so age math is deterministic.
|
|
10
|
+
const NOW = new Date("2026-07-07T12:00:00.000Z");
|
|
11
|
+
const NOW_MS = NOW.getTime();
|
|
12
|
+
|
|
13
|
+
/** dataAsOf `ageSeconds` before NOW. */
|
|
14
|
+
function asOf(ageSeconds: number): string {
|
|
15
|
+
return new Date(NOW_MS - ageSeconds * 1000).toISOString();
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
describe("evaluateManifestFreshness", () => {
|
|
19
|
+
it("serves the table when un-gated (no window)", () => {
|
|
20
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
21
|
+
tableName: "t",
|
|
22
|
+
dataAsOf: asOf(1_000_000),
|
|
23
|
+
};
|
|
24
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it("serves the table when a window is declared but no dataAsOf anchor", () => {
|
|
28
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
29
|
+
tableName: "t",
|
|
30
|
+
freshnessWindowSeconds: 3600,
|
|
31
|
+
freshnessFallback: "live",
|
|
32
|
+
};
|
|
33
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
it("serves the table when fresh (age within window)", () => {
|
|
37
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
38
|
+
tableName: "t",
|
|
39
|
+
dataAsOf: asOf(1800),
|
|
40
|
+
freshnessWindowSeconds: 3600,
|
|
41
|
+
freshnessFallback: "live",
|
|
42
|
+
};
|
|
43
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it("serves the table at the exact window boundary (age == window)", () => {
|
|
47
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
48
|
+
tableName: "t",
|
|
49
|
+
dataAsOf: asOf(3600),
|
|
50
|
+
freshnessWindowSeconds: 3600,
|
|
51
|
+
freshnessFallback: "live",
|
|
52
|
+
};
|
|
53
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
it("drops a stale entry with fallback live (=> serve live)", () => {
|
|
57
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
58
|
+
tableName: "t",
|
|
59
|
+
dataAsOf: asOf(7200),
|
|
60
|
+
freshnessWindowSeconds: 3600,
|
|
61
|
+
freshnessFallback: "live",
|
|
62
|
+
};
|
|
63
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it("keeps a stale entry with fallback stale_ok (=> serve the stale table)", () => {
|
|
67
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
68
|
+
tableName: "t",
|
|
69
|
+
dataAsOf: asOf(7200),
|
|
70
|
+
freshnessWindowSeconds: 3600,
|
|
71
|
+
freshnessFallback: "stale_ok",
|
|
72
|
+
};
|
|
73
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_table");
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it("drops a stale entry with fallback fail (interim: serve live in v1)", () => {
|
|
77
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
78
|
+
tableName: "t",
|
|
79
|
+
dataAsOf: asOf(7200),
|
|
80
|
+
freshnessWindowSeconds: 3600,
|
|
81
|
+
freshnessFallback: "fail",
|
|
82
|
+
};
|
|
83
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
84
|
+
});
|
|
85
|
+
|
|
86
|
+
it("drops a stale entry with no declared fallback (=> serve live)", () => {
|
|
87
|
+
const entry: FreshnessAwareManifestEntry = {
|
|
88
|
+
tableName: "t",
|
|
89
|
+
dataAsOf: asOf(7200),
|
|
90
|
+
freshnessWindowSeconds: 3600,
|
|
91
|
+
};
|
|
92
|
+
expect(evaluateManifestFreshness(entry, NOW)).toBe("serve_live");
|
|
93
|
+
});
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
describe("filterFreshManifest", () => {
|
|
97
|
+
it("includes fresh/un-gated/stale_ok and drops stale live/fail", () => {
|
|
98
|
+
const entries: FreshnessManifest = {
|
|
99
|
+
fresh: {
|
|
100
|
+
tableName: "fresh_t",
|
|
101
|
+
dataAsOf: asOf(1800),
|
|
102
|
+
freshnessWindowSeconds: 3600,
|
|
103
|
+
freshnessFallback: "live",
|
|
104
|
+
},
|
|
105
|
+
ungated: { tableName: "ungated_t", dataAsOf: asOf(999999) },
|
|
106
|
+
staleLive: {
|
|
107
|
+
tableName: "stale_live_t",
|
|
108
|
+
dataAsOf: asOf(7200),
|
|
109
|
+
freshnessWindowSeconds: 3600,
|
|
110
|
+
freshnessFallback: "live",
|
|
111
|
+
},
|
|
112
|
+
staleOk: {
|
|
113
|
+
tableName: "stale_ok_t",
|
|
114
|
+
dataAsOf: asOf(7200),
|
|
115
|
+
freshnessWindowSeconds: 3600,
|
|
116
|
+
freshnessFallback: "stale_ok",
|
|
117
|
+
},
|
|
118
|
+
staleFail: {
|
|
119
|
+
tableName: "stale_fail_t",
|
|
120
|
+
dataAsOf: asOf(7200),
|
|
121
|
+
freshnessWindowSeconds: 3600,
|
|
122
|
+
freshnessFallback: "fail",
|
|
123
|
+
},
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const { manifest } = filterFreshManifest(entries, NOW);
|
|
127
|
+
expect(Object.keys(manifest).sort()).toEqual([
|
|
128
|
+
"fresh",
|
|
129
|
+
"staleOk",
|
|
130
|
+
"ungated",
|
|
131
|
+
]);
|
|
132
|
+
expect(manifest.fresh).toEqual({ tableName: "fresh_t" });
|
|
133
|
+
expect(manifest.staleOk).toEqual({ tableName: "stale_ok_t" });
|
|
134
|
+
expect(manifest.ungated).toEqual({ tableName: "ungated_t" });
|
|
135
|
+
});
|
|
136
|
+
|
|
137
|
+
it("reports the earliest future window-crossing as nextStaleSince", () => {
|
|
138
|
+
const entries: FreshnessManifest = {
|
|
139
|
+
a: {
|
|
140
|
+
tableName: "a_t",
|
|
141
|
+
dataAsOf: asOf(1800), // window 3600 => flips at NOW + 1800s
|
|
142
|
+
freshnessWindowSeconds: 3600,
|
|
143
|
+
freshnessFallback: "live",
|
|
144
|
+
},
|
|
145
|
+
b: {
|
|
146
|
+
tableName: "b_t",
|
|
147
|
+
dataAsOf: asOf(1000), // window 3600 => flips at NOW + 2600s (later)
|
|
148
|
+
freshnessWindowSeconds: 3600,
|
|
149
|
+
freshnessFallback: "live",
|
|
150
|
+
},
|
|
151
|
+
};
|
|
152
|
+
|
|
153
|
+
const { nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
154
|
+
// Earliest flip is entry a: dataAsOf(a) + window.
|
|
155
|
+
expect(nextStaleSince).toBe(NOW_MS - 1800 * 1000 + 3600 * 1000);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
it("kept stale_ok entries do not set a future nextStaleSince", () => {
|
|
159
|
+
const entries: FreshnessManifest = {
|
|
160
|
+
staleOk: {
|
|
161
|
+
tableName: "stale_ok_t",
|
|
162
|
+
dataAsOf: asOf(7200),
|
|
163
|
+
freshnessWindowSeconds: 3600,
|
|
164
|
+
freshnessFallback: "stale_ok",
|
|
165
|
+
},
|
|
166
|
+
};
|
|
167
|
+
const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
168
|
+
expect(Object.keys(manifest)).toEqual(["staleOk"]);
|
|
169
|
+
// Its window crossing is already in the past, so nothing schedules a
|
|
170
|
+
// future recompute.
|
|
171
|
+
expect(nextStaleSince).toBeNull();
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
it("returns null nextStaleSince when nothing is gated", () => {
|
|
175
|
+
const entries: FreshnessManifest = {
|
|
176
|
+
a: { tableName: "a_t" },
|
|
177
|
+
b: { tableName: "b_t", dataAsOf: asOf(10) },
|
|
178
|
+
};
|
|
179
|
+
const { manifest, nextStaleSince } = filterFreshManifest(entries, NOW);
|
|
180
|
+
expect(Object.keys(manifest).sort()).toEqual(["a", "b"]);
|
|
181
|
+
expect(nextStaleSince).toBeNull();
|
|
182
|
+
});
|
|
183
|
+
});
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-query freshness gate for persisted sources (persistence.md §9.3, §14
|
|
3
|
+
* Phase B). A query on a `#@ persist` source may use its materialized table
|
|
4
|
+
* only while that table is within its declared freshness window; otherwise the
|
|
5
|
+
* query does what the source's `fallback` says.
|
|
6
|
+
*
|
|
7
|
+
* Two settled design principles anchor this (do not re-derive them the easy
|
|
8
|
+
* way):
|
|
9
|
+
*
|
|
10
|
+
* 1. Enforce per query, not at bind. Filtering stale entries once when the
|
|
11
|
+
* manifest is bound would let a table that crosses its window while the
|
|
12
|
+
* package stays loaded keep serving stale. The serve path re-evaluates
|
|
13
|
+
* freshness on every query.
|
|
14
|
+
* 2. Clock on `dataAsOf`, not build completion. The window is a deadline on
|
|
15
|
+
* *data* age; the anchor is the artifact's data-as-of instant (snapshot
|
|
16
|
+
* start) supplied by the control plane. `age = now - dataAsOf`.
|
|
17
|
+
*
|
|
18
|
+
* Scope: `live`/`stale_ok` are fully publisher-side. `fail` degrades to
|
|
19
|
+
* serve-live in this first cut (the entry is dropped like `live`); the airtight
|
|
20
|
+
* `fail` path — which must *error* a query that joins a stale persist source —
|
|
21
|
+
* is a gated forked-malloy follow-up and is not built here.
|
|
22
|
+
*/
|
|
23
|
+
|
|
24
|
+
import {
|
|
25
|
+
BuildManifest,
|
|
26
|
+
FreshnessAwareManifestEntry,
|
|
27
|
+
FreshnessManifest,
|
|
28
|
+
} from "../storage/DatabaseInterface";
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* What to do with a single manifest entry at query time.
|
|
32
|
+
* - `serve_table`: substitute the materialized table (include `{ tableName }`).
|
|
33
|
+
* - `serve_live`: drop the entry so `strict:false` falls through to live SQL.
|
|
34
|
+
*
|
|
35
|
+
* `fail` collapses to `serve_live` in v1 (see module note).
|
|
36
|
+
*/
|
|
37
|
+
export type FreshnessDecision = "serve_table" | "serve_live";
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Evaluate one entry against `now`. An entry with no `freshnessWindowSeconds`
|
|
41
|
+
* is un-gated (always serve the table). A fresh entry (age within window) is
|
|
42
|
+
* served. A stale entry serves its table only under `stale_ok`; `live` and
|
|
43
|
+
* `fail` both drop to live in this first cut.
|
|
44
|
+
*/
|
|
45
|
+
export function evaluateManifestFreshness(
|
|
46
|
+
entry: FreshnessAwareManifestEntry,
|
|
47
|
+
now: Date,
|
|
48
|
+
): FreshnessDecision {
|
|
49
|
+
// Un-gated: no declared window ⇒ never stale. Also the rollout-safe default
|
|
50
|
+
// before the control plane starts stamping the freshness fields.
|
|
51
|
+
if (entry.freshnessWindowSeconds == null) return "serve_table";
|
|
52
|
+
// A window without an anchor can't be evaluated; treat as un-gated rather
|
|
53
|
+
// than tripping every query (matches "absent ⇒ never stale").
|
|
54
|
+
if (entry.dataAsOf == null) return "serve_table";
|
|
55
|
+
const dataAsOfMs = Date.parse(entry.dataAsOf);
|
|
56
|
+
if (Number.isNaN(dataAsOfMs)) return "serve_table";
|
|
57
|
+
const ageSeconds = (now.getTime() - dataAsOfMs) / 1000;
|
|
58
|
+
const stale = ageSeconds > entry.freshnessWindowSeconds;
|
|
59
|
+
if (!stale) return "serve_table";
|
|
60
|
+
return entry.freshnessFallback === "stale_ok" ? "serve_table" : "serve_live";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* The instant a currently-fresh, gated entry will cross its window
|
|
65
|
+
* (`dataAsOf + window`), or `null` when the entry has no evaluable window
|
|
66
|
+
* (un-gated). Used to compute the next memo-invalidation instant.
|
|
67
|
+
*/
|
|
68
|
+
function staleSinceMs(entry: FreshnessAwareManifestEntry): number | null {
|
|
69
|
+
if (entry.freshnessWindowSeconds == null || entry.dataAsOf == null) {
|
|
70
|
+
return null;
|
|
71
|
+
}
|
|
72
|
+
const dataAsOfMs = Date.parse(entry.dataAsOf);
|
|
73
|
+
if (Number.isNaN(dataAsOfMs)) return null;
|
|
74
|
+
return dataAsOfMs + entry.freshnessWindowSeconds * 1000;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Compute the freshness-filtered Malloy build manifest for `now` from the full
|
|
79
|
+
* retained wire entries.
|
|
80
|
+
*
|
|
81
|
+
* Returns the `sourceEntityId -> { tableName }` map of entries that should
|
|
82
|
+
* substitute their materialized table (fresh, un-gated, or stale-`stale_ok`),
|
|
83
|
+
* plus `nextStaleSince`: the earliest future instant a currently-included,
|
|
84
|
+
* gated entry will cross its window. Because age only ever grows, the
|
|
85
|
+
* fresh→stale transition is one-way, so a single `nextStaleSince` instant is a
|
|
86
|
+
* sufficient memo-invalidation deadline — no per-entry heap is needed. `null`
|
|
87
|
+
* means no included entry has an evaluable window, so the result never changes
|
|
88
|
+
* until the entries are replaced (rebind).
|
|
89
|
+
*/
|
|
90
|
+
export function filterFreshManifest(
|
|
91
|
+
entries: FreshnessManifest,
|
|
92
|
+
now: Date,
|
|
93
|
+
): { manifest: BuildManifest["entries"]; nextStaleSince: number | null } {
|
|
94
|
+
const manifest: BuildManifest["entries"] = {};
|
|
95
|
+
let nextStaleSince: number | null = null;
|
|
96
|
+
const nowMs = now.getTime();
|
|
97
|
+
|
|
98
|
+
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
99
|
+
if (evaluateManifestFreshness(entry, now) === "serve_live") continue;
|
|
100
|
+
manifest[sourceEntityId] = { tableName: entry.tableName };
|
|
101
|
+
|
|
102
|
+
// Only currently-fresh gated entries can flip later; a kept stale
|
|
103
|
+
// `stale_ok` entry (staleSince already in the past) never changes again.
|
|
104
|
+
const flipAt = staleSinceMs(entry);
|
|
105
|
+
if (flipAt != null && flipAt > nowMs) {
|
|
106
|
+
nextStaleSince =
|
|
107
|
+
nextStaleSince == null ? flipAt : Math.min(nextStaleSince, flipAt);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
return { manifest, nextStaleSince };
|
|
112
|
+
}
|
|
@@ -46,6 +46,39 @@ describe("fetchManifestEntries", () => {
|
|
|
46
46
|
});
|
|
47
47
|
});
|
|
48
48
|
|
|
49
|
+
it("carries the control-plane freshness fields verbatim", async () => {
|
|
50
|
+
const dataAsOf = "2026-07-07T00:00:00.000Z";
|
|
51
|
+
const file = await writeManifest({
|
|
52
|
+
entries: {
|
|
53
|
+
gated: {
|
|
54
|
+
sourceEntityId: "gated",
|
|
55
|
+
physicalTableName: "schema.gated_mz",
|
|
56
|
+
dataAsOf,
|
|
57
|
+
freshnessWindowSeconds: 3600,
|
|
58
|
+
freshnessFallback: "stale_ok",
|
|
59
|
+
},
|
|
60
|
+
ungated: {
|
|
61
|
+
sourceEntityId: "ungated",
|
|
62
|
+
physicalTableName: "schema.ungated_mz",
|
|
63
|
+
},
|
|
64
|
+
},
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const entries = await fetchManifestEntries(file);
|
|
68
|
+
|
|
69
|
+
// Filter-free: freshness fields are retained for the serve-path gate; a
|
|
70
|
+
// window-less entry is left un-gated.
|
|
71
|
+
expect(entries).toEqual({
|
|
72
|
+
gated: {
|
|
73
|
+
tableName: "schema.gated_mz",
|
|
74
|
+
dataAsOf,
|
|
75
|
+
freshnessWindowSeconds: 3600,
|
|
76
|
+
freshnessFallback: "stale_ok",
|
|
77
|
+
},
|
|
78
|
+
ungated: { tableName: "schema.ungated_mz" },
|
|
79
|
+
});
|
|
80
|
+
});
|
|
81
|
+
|
|
49
82
|
it("reads via a file:// URI", async () => {
|
|
50
83
|
const file = await writeManifest({
|
|
51
84
|
entries: { b1: { sourceEntityId: "b1", physicalTableName: "t1" } },
|
|
@@ -4,7 +4,7 @@ import * as fs from "fs/promises";
|
|
|
4
4
|
import { fileURLToPath } from "url";
|
|
5
5
|
import { components } from "../api";
|
|
6
6
|
import { logger } from "../logger";
|
|
7
|
-
import {
|
|
7
|
+
import { FreshnessManifest } from "../storage/DatabaseInterface";
|
|
8
8
|
|
|
9
9
|
type WireBuildManifest = components["schemas"]["BuildManifest"];
|
|
10
10
|
|
|
@@ -58,14 +58,18 @@ async function readManifestBytes(uri: string): Promise<string> {
|
|
|
58
58
|
|
|
59
59
|
/**
|
|
60
60
|
* Fetch and parse the control-plane-computed build manifest at `uri`, returning
|
|
61
|
-
* the
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
61
|
+
* the full wire binding map (`sourceEntityId -> { tableName, dataAsOf,
|
|
62
|
+
* freshnessWindowSeconds, freshnessFallback }`). The wire manifest keys physical
|
|
63
|
+
* tables under `physicalTableName`; the Malloy runtime consumes `tableName`, so
|
|
64
|
+
* we translate that field here but otherwise carry the control-plane freshness
|
|
65
|
+
* fields verbatim so the serve path can gate `age vs window` per query. Entries
|
|
66
|
+
* missing a physical table are skipped. This is a pure fetch + parse: it does
|
|
67
|
+
* **not** filter on freshness (that happens per query on the serve path). Throws
|
|
68
|
+
* if the URI can't be read or parsed.
|
|
65
69
|
*/
|
|
66
70
|
export async function fetchManifestEntries(
|
|
67
71
|
uri: string,
|
|
68
|
-
): Promise<
|
|
72
|
+
): Promise<FreshnessManifest> {
|
|
69
73
|
const raw = await readManifestBytes(uri);
|
|
70
74
|
|
|
71
75
|
let parsed: WireBuildManifest;
|
|
@@ -79,7 +83,7 @@ export async function fetchManifestEntries(
|
|
|
79
83
|
);
|
|
80
84
|
}
|
|
81
85
|
|
|
82
|
-
const entries:
|
|
86
|
+
const entries: FreshnessManifest = {};
|
|
83
87
|
for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
|
|
84
88
|
const physicalTableName = entry?.physicalTableName;
|
|
85
89
|
if (!physicalTableName) {
|
|
@@ -89,7 +93,12 @@ export async function fetchManifestEntries(
|
|
|
89
93
|
});
|
|
90
94
|
continue;
|
|
91
95
|
}
|
|
92
|
-
entries[sourceEntityId] = {
|
|
96
|
+
entries[sourceEntityId] = {
|
|
97
|
+
tableName: physicalTableName,
|
|
98
|
+
dataAsOf: entry.dataAsOf,
|
|
99
|
+
freshnessWindowSeconds: entry.freshnessWindowSeconds,
|
|
100
|
+
freshnessFallback: entry.freshnessFallback,
|
|
101
|
+
};
|
|
93
102
|
}
|
|
94
103
|
return entries;
|
|
95
104
|
}
|
|
@@ -7,14 +7,17 @@ import { Environment } from "./environment";
|
|
|
7
7
|
import type { Package } from "./package";
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
* The publish gate for the package-level materialization cron
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
10
|
+
* The publish gate for the package-level materialization cron. At Phase B the
|
|
11
|
+
* package-level cron (`materialization.schedule`) is replaced outright by
|
|
12
|
+
* `materialization.freshness.window` (persistence.md §9.4), so ANY declared cron
|
|
13
|
+
* is rejected regardless of the sources' sharing. The `sharing=private`
|
|
14
|
+
* carve-out arrives whole with Phase A (persistence-phase-b-design.md §3.4);
|
|
15
|
+
* until then there is no private artifact for a cron to own, so the B→A rule is
|
|
16
|
+
* reject-all. These tests run a real `Environment` + `Package.create` over temp
|
|
17
|
+
* dirs, so they also prove end-to-end that the pinned compiler ACCEPTS
|
|
18
|
+
* `sharing=` / `refresh=` keys in the `#@ persist` annotation and that the
|
|
19
|
+
* values survive to the wire build plan verbatim (unset stays null — never
|
|
20
|
+
* defaulted to "shared").
|
|
18
21
|
*/
|
|
19
22
|
describe("materialization cron gate", () => {
|
|
20
23
|
let rootDir: string;
|
|
@@ -99,29 +102,31 @@ source: b is duckdb.sql("SELECT 2 as x")
|
|
|
99
102
|
);
|
|
100
103
|
|
|
101
104
|
it(
|
|
102
|
-
"rejects
|
|
105
|
+
"rejects any declared cron, pointing at freshness.window",
|
|
103
106
|
async () => {
|
|
104
107
|
const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
|
|
105
108
|
|
|
109
|
+
// One actionable message for the whole package — the B→A rule is
|
|
110
|
+
// reject-all, not per-source.
|
|
106
111
|
const warnings = pkg.scheduleWarnings();
|
|
107
|
-
|
|
108
|
-
// source is not flagged.
|
|
109
|
-
expect(warnings).toHaveLength(2);
|
|
112
|
+
expect(warnings).toHaveLength(1);
|
|
110
113
|
const joined = pkg.formatInvalidSchedule();
|
|
111
|
-
expect(joined).toContain("
|
|
112
|
-
expect(joined).toContain("'unspecified' resolves to unset");
|
|
113
|
-
expect(joined).not.toContain("'priv'");
|
|
114
|
+
expect(joined).toContain("materialization.schedule");
|
|
114
115
|
expect(joined).toContain("materialization.freshness.window");
|
|
115
116
|
},
|
|
116
117
|
{ timeout: 30000 },
|
|
117
118
|
);
|
|
118
119
|
|
|
119
120
|
it(
|
|
120
|
-
"
|
|
121
|
+
"rejects a declared cron even when every persist source is private",
|
|
121
122
|
async () => {
|
|
123
|
+
// The sharing=private carve-out arrives with Phase A; during B→A even
|
|
124
|
+
// an all-private package's cron is rejected outright.
|
|
122
125
|
const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
|
|
123
|
-
expect(pkg.scheduleWarnings()).
|
|
124
|
-
expect(pkg.formatInvalidSchedule()).
|
|
126
|
+
expect(pkg.scheduleWarnings()).toHaveLength(1);
|
|
127
|
+
expect(pkg.formatInvalidSchedule()).toContain(
|
|
128
|
+
"materialization.freshness.window",
|
|
129
|
+
);
|
|
125
130
|
},
|
|
126
131
|
{ timeout: 30000 },
|
|
127
132
|
);
|
|
@@ -23,6 +23,7 @@ import {
|
|
|
23
23
|
BuildManifest,
|
|
24
24
|
BuildManifestResult,
|
|
25
25
|
BuildPlan,
|
|
26
|
+
FreshnessManifest,
|
|
26
27
|
Materialization,
|
|
27
28
|
MaterializationStatus,
|
|
28
29
|
MaterializationUpdate,
|
|
@@ -480,12 +481,16 @@ export class MaterializationService {
|
|
|
480
481
|
environment: {
|
|
481
482
|
reloadAllModelsForPackage(
|
|
482
483
|
packageName: string,
|
|
483
|
-
manifest:
|
|
484
|
+
manifest: FreshnessManifest,
|
|
484
485
|
): Promise<void>;
|
|
485
486
|
},
|
|
486
487
|
packageName: string,
|
|
487
488
|
entries: Record<string, ManifestEntry>,
|
|
488
489
|
): Promise<void> {
|
|
490
|
+
// The post-build auto-load binds tableName-only entries: the control plane
|
|
491
|
+
// stamps freshness (dataAsOf/window/fallback) on the wire manifest it
|
|
492
|
+
// distributes, not on this in-memory post-build load, so these sources are
|
|
493
|
+
// bound un-gated (always serve the freshly-built table).
|
|
489
494
|
const manifestEntries: BuildManifest["entries"] = {};
|
|
490
495
|
for (const [sourceEntityId, entry] of Object.entries(entries)) {
|
|
491
496
|
if (entry.physicalTableName) {
|
package/src/service/model.ts
CHANGED
|
@@ -164,6 +164,14 @@ export class Model {
|
|
|
164
164
|
exploresDeclared: boolean;
|
|
165
165
|
isQueryEntryPoint: boolean;
|
|
166
166
|
} = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
|
|
167
|
+
/** Per-query freshness resolver, pushed down by the owning Package (see
|
|
168
|
+
* Package.wireFreshnessResolvers). Returns the freshness-filtered build
|
|
169
|
+
* manifest for the serve path — threaded into Malloy's per-query
|
|
170
|
+
* `buildManifest` override so a persist source only routes to its
|
|
171
|
+
* materialized table while within its declared freshness window. Undefined
|
|
172
|
+
* (or returning undefined) means no override: the runtime-baked manifest
|
|
173
|
+
* applies, which serves live when unbound. */
|
|
174
|
+
private freshnessResolver?: () => BuildManifest["entries"] | undefined;
|
|
167
175
|
private meter = publisherMeter();
|
|
168
176
|
private queryExecutionHistogram = this.meter.createHistogram(
|
|
169
177
|
"malloy_model_query_duration",
|
|
@@ -743,6 +751,29 @@ export class Model {
|
|
|
743
751
|
this.discoveryCurationEnabled = enabled;
|
|
744
752
|
}
|
|
745
753
|
|
|
754
|
+
/**
|
|
755
|
+
* Set by the owning Package (see Package.wireFreshnessResolvers). Supplies
|
|
756
|
+
* the freshness-filtered build manifest the serve path threads into Malloy's
|
|
757
|
+
* per-query `buildManifest` override so stale persist sources fall back per
|
|
758
|
+
* their declared policy. See {@link resolveFreshBuildManifest}.
|
|
759
|
+
*/
|
|
760
|
+
public setFreshnessResolver(
|
|
761
|
+
resolver: () => BuildManifest["entries"] | undefined,
|
|
762
|
+
): void {
|
|
763
|
+
this.freshnessResolver = resolver;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* The freshness-filtered build manifest for this query, or undefined when the
|
|
768
|
+
* package is unbound / has no resolver (⇒ no per-query override; the runtime
|
|
769
|
+
* serves live). Evaluated per call so a table that crosses its window while
|
|
770
|
+
* the package stays loaded is gated on the very next query.
|
|
771
|
+
*/
|
|
772
|
+
private resolveFreshBuildManifest(): BuildManifest | undefined {
|
|
773
|
+
const entries = this.freshnessResolver?.();
|
|
774
|
+
return entries ? { entries, strict: false } : undefined;
|
|
775
|
+
}
|
|
776
|
+
|
|
746
777
|
public getSources(): ApiSource[] | undefined {
|
|
747
778
|
return this.curateForDiscovery(this.sources);
|
|
748
779
|
}
|
|
@@ -1273,8 +1304,14 @@ export class Model {
|
|
|
1273
1304
|
|
|
1274
1305
|
const maxRows = getMaxQueryRows();
|
|
1275
1306
|
const maxBytes = getMaxResponseBytes();
|
|
1307
|
+
// Per-query freshness gate (persistence.md §9.3): resolve the
|
|
1308
|
+
// freshness-filtered manifest once and thread it into both the prepare
|
|
1309
|
+
// (for the row limit) and the run so a stale persist source falls back per
|
|
1310
|
+
// its declared policy — and prep/run agree on the same substitution.
|
|
1311
|
+
const buildManifest = this.resolveFreshBuildManifest();
|
|
1276
1312
|
const rowLimit = resolveModelQueryRowLimit(
|
|
1277
|
-
(await runnable.getPreparedResult({ givens }))
|
|
1313
|
+
(await runnable.getPreparedResult({ givens, buildManifest }))
|
|
1314
|
+
.resultExplore.limit,
|
|
1278
1315
|
{ defaultLimit: getDefaultQueryRowLimit(), maxRows },
|
|
1279
1316
|
);
|
|
1280
1317
|
const endTime = performance.now();
|
|
@@ -1282,7 +1319,12 @@ export class Model {
|
|
|
1282
1319
|
|
|
1283
1320
|
let queryResults;
|
|
1284
1321
|
try {
|
|
1285
|
-
queryResults = await runnable.run({
|
|
1322
|
+
queryResults = await runnable.run({
|
|
1323
|
+
rowLimit,
|
|
1324
|
+
givens,
|
|
1325
|
+
abortSignal,
|
|
1326
|
+
buildManifest,
|
|
1327
|
+
});
|
|
1286
1328
|
} catch (error) {
|
|
1287
1329
|
// Record error metrics
|
|
1288
1330
|
const errorEndTime = performance.now();
|
|
@@ -1521,9 +1563,16 @@ export class Model {
|
|
|
1521
1563
|
|
|
1522
1564
|
const cellMaxRows = getMaxQueryRows();
|
|
1523
1565
|
const cellMaxBytes = getMaxResponseBytes();
|
|
1566
|
+
// Per-query freshness gate (see getQueryResults): the same
|
|
1567
|
+
// freshness-filtered manifest gates notebook-cell queries.
|
|
1568
|
+
const buildManifest = this.resolveFreshBuildManifest();
|
|
1524
1569
|
const rowLimit = resolveModelQueryRowLimit(
|
|
1525
|
-
(
|
|
1526
|
-
.
|
|
1570
|
+
(
|
|
1571
|
+
await runnableToExecute.getPreparedResult({
|
|
1572
|
+
givens,
|
|
1573
|
+
buildManifest,
|
|
1574
|
+
})
|
|
1575
|
+
).resultExplore.limit,
|
|
1527
1576
|
{
|
|
1528
1577
|
defaultLimit: getDefaultQueryRowLimit(),
|
|
1529
1578
|
maxRows: cellMaxRows,
|
|
@@ -1533,6 +1582,7 @@ export class Model {
|
|
|
1533
1582
|
rowLimit,
|
|
1534
1583
|
givens,
|
|
1535
1584
|
abortSignal,
|
|
1585
|
+
buildManifest,
|
|
1536
1586
|
});
|
|
1537
1587
|
const query = (await runnableToExecute.getPreparedQuery())._query;
|
|
1538
1588
|
queryName = (query as NamedQueryDef).as || query.name;
|