@malloy-publisher/server 0.0.223 → 0.0.225

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/dist/app/api-doc.yaml +75 -0
  2. package/dist/app/assets/{EnvironmentPage-3wCdljov.js → EnvironmentPage-DYTeXDll.js} +1 -1
  3. package/dist/app/assets/{HomePage-K0GHwqq2.js → HomePage-pDK2BPJY.js} +1 -1
  4. package/dist/app/assets/{LightMode-CwU3kN4I.js → LightMode-C2bwGPY1.js} +1 -1
  5. package/dist/app/assets/{MainPage-CTncHE5T.js → MainPage-WtBulMH_.js} +2 -2
  6. package/dist/app/assets/{MaterializationsPage-DziAOD6-.js → MaterializationsPage-hMgOtflG.js} +1 -1
  7. package/dist/app/assets/{ModelPage-XrS2jXEc.js → ModelPage-B2N5kYII.js} +1 -1
  8. package/dist/app/assets/{PackagePage-BkwgFxG8.js → PackagePage-CEN90nQG.js} +1 -1
  9. package/dist/app/assets/{RouteError-9cIQga6p.js → RouteError-BG2c5Zf0.js} +1 -1
  10. package/dist/app/assets/{ThemeEditorPage-DWiCRfEe.js → ThemeEditorPage-DNfeUwEZ.js} +1 -1
  11. package/dist/app/assets/{WorkbookPage-Ce7FM_Po.js → WorkbookPage-NKI1BhFS.js} +1 -1
  12. package/dist/app/assets/{core-D9Hl0IDY.es-CrO01m4X.js → core-C6anj5c0.es-DDLHqpzt.js} +1 -1
  13. package/dist/app/assets/{index-D2sm5RBA.js → index-C6gZ6sSY.js} +5 -5
  14. package/dist/app/assets/{index-CtQm7kvp.js → index-CzNqKMVl.js} +1 -1
  15. package/dist/app/assets/{index-wJU_kzl2.js → index-DMQtnaf4.js} +2 -2
  16. package/dist/app/assets/{index-Dz8hgkmy.js → index-JXgvyZna.js} +1 -1
  17. package/dist/app/assets/{index-8E2uLeV9.js → index-OEjKNSYb.js} +2 -2
  18. package/dist/app/index.html +1 -1
  19. package/dist/server.mjs +411 -26
  20. package/dist/sshcrypto-8m50vnmb.node +0 -0
  21. package/package.json +12 -12
  22. package/src/controller/materialization.controller.spec.ts +72 -0
  23. package/src/controller/materialization.controller.ts +75 -11
  24. package/src/mcp/skills/skills_bundle.json +1 -1
  25. package/src/service/build_plan.spec.ts +119 -0
  26. package/src/service/build_plan.ts +143 -0
  27. package/src/service/environment.ts +3 -3
  28. package/src/service/freshness.spec.ts +183 -0
  29. package/src/service/freshness.ts +112 -0
  30. package/src/service/manifest_loader.spec.ts +33 -0
  31. package/src/service/manifest_loader.ts +17 -8
  32. package/src/service/materialization_cron_gate.spec.ts +61 -17
  33. package/src/service/materialization_service.spec.ts +42 -0
  34. package/src/service/materialization_service.ts +46 -3
  35. package/src/service/materialization_test_fixtures.ts +47 -14
  36. package/src/service/model.ts +54 -4
  37. package/src/service/package.ts +173 -35
  38. package/src/storage/DatabaseInterface.ts +24 -0
  39. package/tests/fixtures/persist-multi-level/data/orders.csv +5 -0
  40. package/tests/fixtures/persist-multi-level/multi_level.malloy +18 -0
  41. package/tests/fixtures/persist-multi-level/publisher.json +5 -0
  42. package/tests/integration/materialization/freshness_gate.integration.spec.ts +292 -0
  43. package/tests/integration/materialization/reference_manifest.integration.spec.ts +251 -0
@@ -30,9 +30,14 @@ import {
30
30
  import { formatDuration, logger } from "../logger";
31
31
  import { recordBuildPlanComputeDuration } from "../materialization_metrics";
32
32
  import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
33
- import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
33
+ import {
34
+ BuildManifest,
35
+ BuildPlan,
36
+ FreshnessManifest,
37
+ } from "../storage/DatabaseInterface";
34
38
  import { errMessage, ignoreDotfiles } from "../utils";
35
39
  import { computePackageBuildPlan } from "./build_plan";
40
+ import { filterFreshManifest } from "./freshness";
36
41
  import { Model } from "./model";
37
42
  import { assertPersistNamesQuoted } from "./persist_annotation_validation";
38
43
 
@@ -52,6 +57,22 @@ type PackageConnectionInput =
52
57
  | Map<string, Connection>
53
58
  | (() => MalloyConfig);
54
59
 
60
+ /**
61
+ * Project the full wire entries down to the Malloy-runtime binding map
62
+ * (`sourceEntityId -> { tableName }`) used to hydrate models. The freshness
63
+ * fields are dropped here — they gate the serve path per query, not model
64
+ * hydration.
65
+ */
66
+ function toTableNameManifest(
67
+ entries: FreshnessManifest,
68
+ ): BuildManifest["entries"] {
69
+ const out: BuildManifest["entries"] = {};
70
+ for (const [sourceEntityId, entry] of Object.entries(entries)) {
71
+ out[sourceEntityId] = { tableName: entry.tableName };
72
+ }
73
+ return out;
74
+ }
75
+
55
76
  export class Package {
56
77
  private environmentName: string;
57
78
  private packageName: string;
@@ -67,6 +88,22 @@ export class Package {
67
88
  // /status (via getPackageMetadata) so the control plane can confirm a worker
68
89
  // actually bound the distributed manifest instead of inferring it from logs.
69
90
  private buildManifestEntries: BuildManifest["entries"] | undefined;
91
+ // The full wire entries retained at bind (sourceEntityId -> { tableName,
92
+ // dataAsOf, freshnessWindowSeconds, freshnessFallback }). Unlike
93
+ // {@link buildManifestEntries} (the tableName-only projection baked into the
94
+ // hydration runtime and reused by /compile), this keeps the control-plane
95
+ // freshness fields so the serve path can re-evaluate `age vs window` per
96
+ // query. Undefined when the package is serving live (unbound).
97
+ private freshnessEntries: FreshnessManifest | undefined;
98
+ // Memoized freshness-filtered manifest for the serve path. Almost all queries
99
+ // in a window share the same included set, so this is recomputed only when a
100
+ // retained entry actually crosses its window (`validUntil`, the next
101
+ // staleSince) or when a rebind replaces the entries (cleared in
102
+ // recordManifestBinding). validUntil === Infinity means no included entry has
103
+ // an evaluable window, so the result is stable until the next rebind.
104
+ private freshManifestCache:
105
+ | { manifest: BuildManifest["entries"]; validUntil: number }
106
+ | undefined;
70
107
  private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
71
108
  "unbound";
72
109
  private manifestEntryCount = 0;
@@ -419,6 +456,10 @@ export class Package {
419
456
  malloyConfig,
420
457
  );
421
458
  pkg.renderTagWarnings = renderTagWarnings;
459
+ // Install the per-query freshness resolver on the freshly-built models.
460
+ // At create time no manifest is bound yet, so the resolver returns
461
+ // undefined (serve live) until a subsequent bindManifest → reloadAllModels.
462
+ pkg.wireFreshnessResolvers();
422
463
 
423
464
  // Compute the persist build plan off the live (unbound) models, before the
424
465
  // caller binds any configured manifest, so the surfaced plan reflects the
@@ -480,6 +521,17 @@ export class Package {
480
521
  return this.buildPlan;
481
522
  }
482
523
 
524
+ /**
525
+ * The package-level `materialization` config (from malloy-publisher.json),
526
+ * the least-specific layer for resolving per-source freshness/schedule in the
527
+ * build plan. Null when the package declares no policy.
528
+ */
529
+ public getMaterializationConfig(): NonNullable<
530
+ ApiPackage["materialization"]
531
+ > | null {
532
+ return this.packageMetadata.materialization ?? null;
533
+ }
534
+
483
535
  public getPackageMetadata(): ApiPackage {
484
536
  // Overlay the server-computed fields onto the stored metadata: the
485
537
  // explores misconfig warnings (loading is fail-safe — the package still
@@ -523,6 +575,51 @@ export class Package {
523
575
  return this.buildManifestEntries;
524
576
  }
525
577
 
578
+ /**
579
+ * The freshness-filtered build manifest for the serve path, evaluated at
580
+ * `now`. Persistence.md §9.3 Phase B: a query on a `#@ persist` source may
581
+ * use its materialized table only while the table is within its declared
582
+ * freshness window; otherwise it falls back per the entry's `fallback`
583
+ * (`live`/`fail` drop the entry → serve live; `stale_ok` keeps it → serve the
584
+ * stale table). Un-gated entries (no window) always route to the table.
585
+ *
586
+ * Undefined when the package is serving live (unbound) — callers then apply
587
+ * no per-query override and the runtime serves live.
588
+ *
589
+ * Memoized: recomputed only when a retained entry crosses its window
590
+ * (monotonic fresh→stale, so a single `validUntil` deadline suffices) or when
591
+ * a rebind replaces the entries. Cheap O(1) hit on the common path.
592
+ */
593
+ public getFreshBuildManifest(
594
+ now: number = Date.now(),
595
+ ): BuildManifest["entries"] | undefined {
596
+ if (!this.freshnessEntries) return undefined;
597
+ if (this.freshManifestCache && now < this.freshManifestCache.validUntil) {
598
+ return this.freshManifestCache.manifest;
599
+ }
600
+ const { manifest, nextStaleSince } = filterFreshManifest(
601
+ this.freshnessEntries,
602
+ new Date(now),
603
+ );
604
+ this.freshManifestCache = {
605
+ manifest,
606
+ validUntil: nextStaleSince ?? Infinity,
607
+ };
608
+ return manifest;
609
+ }
610
+
611
+ /**
612
+ * Install the per-query freshness resolver on every owned model so the serve
613
+ * path (Model.getQueryResults / executeNotebookCell) applies the
614
+ * freshness-filtered manifest as a per-query Malloy `buildManifest` override.
615
+ * Idempotent; called after (re)building the model set.
616
+ */
617
+ private wireFreshnessResolvers(): void {
618
+ for (const model of this.models.values()) {
619
+ model.setFreshnessResolver(() => this.getFreshBuildManifest());
620
+ }
621
+ }
622
+
526
623
  /**
527
624
  * Record the URI whose manifest is currently bound to the served models. May
528
625
  * differ from `manifestLocation` after an in-memory auto-load following a
@@ -545,13 +642,18 @@ export class Package {
545
642
  * observability (/status). An empty map means the manifest was cleared, so
546
643
  * the package reverts to live (unbound).
547
644
  */
548
- private recordManifestBinding(
549
- buildManifest: BuildManifest["entries"],
550
- ): void {
551
- const count = Object.keys(buildManifest).length;
552
- this.buildManifestEntries = count > 0 ? buildManifest : undefined;
645
+ private recordManifestBinding(entries: FreshnessManifest): void {
646
+ const count = Object.keys(entries).length;
647
+ // Full wire entries (with freshness) drive the per-query serve-path gate;
648
+ // the tableName-only projection is what /compile and /status consume.
649
+ this.freshnessEntries = count > 0 ? entries : undefined;
650
+ this.buildManifestEntries =
651
+ count > 0 ? toTableNameManifest(entries) : undefined;
553
652
  this.manifestEntryCount = count;
554
653
  this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
654
+ // A rebind replaces the entries, so any memoized freshness-filtered
655
+ // manifest is stale — drop it so the next query recomputes.
656
+ this.freshManifestCache = undefined;
555
657
  if (count === 0) {
556
658
  this.boundManifestUri = null;
557
659
  }
@@ -615,33 +717,61 @@ export class Package {
615
717
  }
616
718
 
617
719
  /**
618
- * Publish-gate for the package-level materialization cron (the power tier
619
- * of artifact-anchored scheduling): a declared `materialization.schedule`
620
- * governs every persist source in the package, so it is valid only when
621
- * each source in the compiled build plan resolves to an explicit
622
- * `sharing=private`. A cron over any shared/unset source would let one
623
- * package dictate the refresh cadence of an artifact other packages share —
624
- * those packages declare `materialization.freshness.window` instead and the
625
- * control plane schedules the shared artifact from the tightest window.
626
- * One actionable message per offending source (empty when the cron is
627
- * valid or no cron is declared). Strict at publish (package.controller),
628
- * warn-only at load/reload (loadViaWorker) same split as explores.
720
+ * Publish-gate for materialization crons (Phase A carve-out, persistence.md
721
+ * §9.4). A cron is the power tier of artifact-anchored scheduling and is
722
+ * valid only over `sharing="private"` artifacts; a cron on a shared (or unset
723
+ * shared) artifact is rejected, pointing at `freshness.window` as the
724
+ * shared-tier replacement. Two grains, both enforced off the resolved build
725
+ * plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
726
+ *
727
+ * - Per-source: a source's own `schedule` is valid only when that source
728
+ * resolves to explicit `sharing="private"`.
729
+ * - Package: `materialization.schedule` governs every persist source, so it
730
+ * is valid only when every governed source resolves to explicit
731
+ * `sharing="private"` (a mixed/shared/unset package cannot carry it).
732
+ *
733
+ * Strict at publish (package.controller), warn-only at load/reload
734
+ * (loadViaWorker) — same split as explores.
629
735
  */
630
736
  public scheduleWarnings(): string[] {
631
- const schedule = this.packageMetadata.materialization?.schedule;
632
- if (!schedule) return [];
633
- const sources = Object.values(this.buildPlan?.sources ?? {});
634
- return sources
635
- .filter((source) => source.sharing !== "private")
636
- .map(
637
- (source) =>
638
- `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` +
639
- `persist source to declare '#@ persist ... sharing=private'; source ` +
640
- `'${source.name}' resolves to ${
641
- source.sharing ? `'${source.sharing}'` : "unset"
642
- }. Declare 'materialization.freshness.window' instead (the control plane ` +
643
- `schedules refreshes from it), or mark every persist source sharing=private.`,
644
- );
737
+ const warnings: string[] = [];
738
+ const sources = this.buildPlan?.sources
739
+ ? Object.values(this.buildPlan.sources)
740
+ : [];
741
+
742
+ for (const source of sources) {
743
+ if (source.schedule && source.sharing !== "private") {
744
+ warnings.push(
745
+ `#@ persist source "${source.name}" declares a schedule (cron) but ` +
746
+ `resolves to sharing="${
747
+ source.sharing ?? "shared (unset default)"
748
+ }": a per-source cron is valid only for sharing="private". ` +
749
+ `Declare 'freshness.window' instead (the control plane schedules ` +
750
+ `refreshes from it).`,
751
+ );
752
+ }
753
+ }
754
+
755
+ const packageSchedule = this.packageMetadata.materialization?.schedule;
756
+ if (packageSchedule) {
757
+ const nonPrivate = sources.filter((s) => s.sharing !== "private");
758
+ if (sources.length === 0 || nonPrivate.length > 0) {
759
+ const detail =
760
+ sources.length === 0
761
+ ? "the package declares no persist source for it to govern"
762
+ : `sources [${nonPrivate
763
+ .map((s) => s.name)
764
+ .join(", ")}] resolve to shared/unset`;
765
+ warnings.push(
766
+ `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` +
767
+ `only when every persist source resolves to sharing="private": ` +
768
+ `${detail}. Declare 'materialization.freshness.window' instead ` +
769
+ `(the control plane schedules refreshes from it).`,
770
+ );
771
+ }
772
+ }
773
+
774
+ return warnings;
645
775
  }
646
776
 
647
777
  /** The {@link scheduleWarnings} joined into one string, or "" if none. */
@@ -728,9 +858,11 @@ export class Package {
728
858
  * timeout, pool shutting down) propagate as `ServiceUnavailableError`
729
859
  * — the caller (manifest service) decides how to retry.
730
860
  */
731
- public async reloadAllModels(
732
- buildManifest: BuildManifest["entries"],
733
- ): Promise<void> {
861
+ public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
862
+ // Models are hydrated against the tableName-only projection; the freshness
863
+ // fields gate the serve path per query (via getFreshBuildManifest), not
864
+ // model hydration.
865
+ const buildManifest = toTableNameManifest(entries);
734
866
  const modelPaths = Array.from(this.models.keys());
735
867
  logger.info("Reloading all models with build manifest", {
736
868
  packageName: this.packageName,
@@ -837,7 +969,13 @@ export class Package {
837
969
  this.applyQueryBoundaryToModels();
838
970
  // Remember what we just bound so /compile can route identically and
839
971
  // /status can report the binding. An empty map reverts to live (unbound).
840
- this.recordManifestBinding(buildManifest);
972
+ // Retains the full freshness entries so the serve-path gate can evaluate
973
+ // them per query.
974
+ this.recordManifestBinding(entries);
975
+ // Install the per-query freshness resolver on the rebuilt model set so the
976
+ // serve path applies the freshness-filtered manifest as a per-query
977
+ // override.
978
+ this.wireFreshnessResolvers();
841
979
  // Re-run the fail-safe warning against the refreshed model set: an edit
842
980
  // to publisher.json that introduces a bad entry should surface in the
843
981
  // logs on reload too, not only at initial load (loadViaWorker).
@@ -116,6 +116,7 @@ export type BuildPlan = components["schemas"]["BuildPlan"];
116
116
  export type BuildManifestResult = components["schemas"]["BuildManifest"];
117
117
  export type ManifestEntry = components["schemas"]["ManifestEntry"];
118
118
  export type BuildInstruction = components["schemas"]["BuildInstruction"];
119
+ export type ManifestReference = components["schemas"]["ManifestReference"];
119
120
  export type Realization = components["schemas"]["Realization"];
120
121
 
121
122
  export interface Materialization {
@@ -157,3 +158,26 @@ export interface BuildManifest {
157
158
  entries: Record<string, BuildManifestEntry>;
158
159
  strict?: boolean;
159
160
  }
161
+
162
+ /**
163
+ * A wire manifest entry carrying the control-plane freshness fields alongside
164
+ * the physical `tableName`. This is what the publisher retains at bind so the
165
+ * serve path can re-evaluate `age vs window` per query (see
166
+ * {@link ../service/freshness}). A `{ tableName }`-only value is structurally
167
+ * assignable here (the freshness fields are optional), so callers that only
168
+ * know a table name — e.g. the post-build auto-load — bind un-gated entries.
169
+ *
170
+ * - `dataAsOf` the artifact's data-as-of instant (snapshot start); the age
171
+ * anchor. `age = now - dataAsOf`. Absent ⇒ never stale.
172
+ * - `freshnessWindowSeconds` the version's effective window. Absent ⇒ un-gated.
173
+ * - `freshnessFallback` per-version read-time policy when stale.
174
+ */
175
+ export interface FreshnessAwareManifestEntry {
176
+ tableName: string;
177
+ dataAsOf?: string;
178
+ freshnessWindowSeconds?: number;
179
+ freshnessFallback?: "live" | "stale_ok" | "fail";
180
+ }
181
+
182
+ /** Full (unfiltered) manifest map keyed by sourceEntityId. */
183
+ export type FreshnessManifest = Record<string, FreshnessAwareManifestEntry>;
@@ -0,0 +1,5 @@
1
+ category,amount
2
+ electronics,100
3
+ clothing,50
4
+ electronics,200
5
+ clothing,75
@@ -0,0 +1,18 @@
1
+ ##! experimental.persistence
2
+
3
+ source: raw_orders is duckdb.table('data/orders.csv')
4
+
5
+ // Upstream persist source: an aggregate over the base CSV.
6
+ #@ persist name="orders_base"
7
+ source: orders_base is raw_orders -> {
8
+ group_by: category
9
+ aggregate: total_amount is amount.sum()
10
+ }
11
+
12
+ // Downstream persist source: references the persist upstream `orders_base`.
13
+ // A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
14
+ // materialized table (via referenceManifest) rather than recompute it live.
15
+ #@ persist name="orders_rollup"
16
+ source: orders_rollup is orders_base -> {
17
+ aggregate: category_count is count()
18
+ }
@@ -0,0 +1,5 @@
1
+ {
2
+ "name": "persist-multi-level",
3
+ "version": "1.0.0",
4
+ "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
5
+ }
@@ -0,0 +1,292 @@
1
+ /// <reference types="bun-types" />
2
+
3
+ import { afterAll, beforeAll, describe, expect, it } from "bun:test";
4
+ import * as fsp from "fs/promises";
5
+ import os from "os";
6
+ import path from "path";
7
+ import { fileURLToPath } from "url";
8
+ import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
9
+
10
+ const __filename = fileURLToPath(import.meta.url);
11
+ const __dirname = path.dirname(__filename);
12
+
13
+ const PROJECT_NAME = "freshness-gate-project";
14
+ const PACKAGE_NAME = "persist-test";
15
+ const MODEL_PATH = "persist_test.malloy";
16
+ const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
17
+
18
+ /**
19
+ * Covers the per-query freshness gate (persistence.md §9.3, §14 Phase B): a
20
+ * query on a `#@ persist` source uses its materialized table only while the
21
+ * table is within its declared freshness window; otherwise it falls back per the
22
+ * entry's declared `fallback`.
23
+ *
24
+ * - fresh (or un-gated) → routes to the materialized table
25
+ * - stale + fallback live → serves live SQL (entry dropped)
26
+ * - stale + fallback stale_ok→ serves the stale table
27
+ * - a fresh entry that crosses its window → serves live on the NEXT query,
28
+ * with no rebind (proves per-query re-evaluation, not bind-time filtering)
29
+ *
30
+ * The manifest is bound once physically (auto-run builds the table), then a
31
+ * manifest URI keyed by the source's real sourceEntityId routes served queries.
32
+ * `executedSql()` is the routing evidence: it scans `order_summary` when routed
33
+ * to the table, and `data/orders.csv` when serving live.
34
+ */
35
+ describe("Per-query freshness gate (E2E)", () => {
36
+ let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
37
+ let baseUrl: string;
38
+ let tmpDir: string;
39
+ let sourceEntityId: string;
40
+
41
+ beforeAll(async () => {
42
+ env = await startRestE2E();
43
+ baseUrl = env.baseUrl;
44
+ tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "freshness-gate-"));
45
+
46
+ const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
47
+ const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
48
+ method: "POST",
49
+ headers: { "Content-Type": "application/json" },
50
+ body: JSON.stringify({
51
+ name: PROJECT_NAME,
52
+ packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
53
+ connections: [],
54
+ }),
55
+ });
56
+ if (!createRes.ok) {
57
+ throw new Error(
58
+ `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
59
+ );
60
+ }
61
+
62
+ const deadline = Date.now() + 30_000;
63
+ while (Date.now() < deadline) {
64
+ const res = await fetch(`${baseUrl}${API}`);
65
+ if (res.ok) break;
66
+ await new Promise((r) => setTimeout(r, 500));
67
+ }
68
+
69
+ // Physically build the persist source (auto-run self-assigns
70
+ // physicalTableName = "order_summary"), then revert to live so the
71
+ // manifest-URI bind is the only thing that can route the served query.
72
+ await buildTableThenRevertToLive();
73
+ sourceEntityId = await orderSummarySourceEntityId();
74
+ }, 120_000);
75
+
76
+ afterAll(async () => {
77
+ if (baseUrl) {
78
+ await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
79
+ method: "DELETE",
80
+ }).catch(() => {});
81
+ }
82
+ await env?.stop();
83
+ await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
84
+ env = null;
85
+ });
86
+
87
+ function url(p: string): string {
88
+ return `${baseUrl}${API}${p}`;
89
+ }
90
+
91
+ const ROUTING_QUERY = "run: order_summary -> { aggregate: c is count() }";
92
+
93
+ /** The SQL the served query actually compiled to (routing evidence). */
94
+ async function executedSql(): Promise<string> {
95
+ const res = await fetch(`${baseUrl}${API}/models/${MODEL_PATH}/query`, {
96
+ method: "POST",
97
+ headers: { "Content-Type": "application/json" },
98
+ body: JSON.stringify({ query: ROUTING_QUERY }),
99
+ });
100
+ expect(res.status).toBe(200);
101
+ const body = (await res.json()) as { result: string };
102
+ return (JSON.parse(body.result) as { sql: string }).sql;
103
+ }
104
+
105
+ async function orderSummarySourceEntityId(): Promise<string> {
106
+ const res = await fetch(url(""));
107
+ expect(res.status).toBe(200);
108
+ const pkg = (await res.json()) as {
109
+ buildPlan?: { sources?: Record<string, { sourceEntityId?: string }> };
110
+ };
111
+ const sources = pkg.buildPlan?.sources ?? {};
112
+ const id = Object.values(sources)[0]?.sourceEntityId;
113
+ expect(typeof id).toBe("string");
114
+ return id as string;
115
+ }
116
+
117
+ async function pollUntilTerminal(
118
+ id: string,
119
+ timeoutMs = 90_000,
120
+ ): Promise<Record<string, unknown>> {
121
+ const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
122
+ const deadline = Date.now() + timeoutMs;
123
+ while (Date.now() < deadline) {
124
+ const res = await fetch(url(`/materializations/${id}`));
125
+ expect(res.status).toBe(200);
126
+ const data = (await res.json()) as Record<string, unknown>;
127
+ if (terminal.includes(data.status as string)) return data;
128
+ await new Promise((r) => setTimeout(r, 250));
129
+ }
130
+ throw new Error(`Materialization ${id} did not reach a terminal state`);
131
+ }
132
+
133
+ async function buildTableThenRevertToLive(): Promise<void> {
134
+ const createRes = await fetch(url("/materializations"), {
135
+ method: "POST",
136
+ headers: { "Content-Type": "application/json" },
137
+ body: JSON.stringify({}),
138
+ });
139
+ expect(createRes.status).toBe(201);
140
+ const { id } = (await createRes.json()) as { id: string };
141
+ const built = await pollUntilTerminal(id);
142
+ expect(built.status).toBe("MANIFEST_FILE_READY");
143
+ await fetch(url("?reload=true"));
144
+ await fetch(url(`/materializations/${id}`), { method: "DELETE" });
145
+ }
146
+
147
+ /** Write a CP-shaped manifest carrying the freshness fields. */
148
+ async function writeFreshnessManifest(opts: {
149
+ dataAsOf: string;
150
+ freshnessWindowSeconds: number;
151
+ freshnessFallback: "live" | "stale_ok" | "fail";
152
+ }): Promise<string> {
153
+ const file = path.join(
154
+ tmpDir,
155
+ `freshness-${Date.now()}-${Math.random()}.json`,
156
+ );
157
+ await fsp.writeFile(
158
+ file,
159
+ JSON.stringify({
160
+ builtAt: new Date().toISOString(),
161
+ strict: false,
162
+ entries: {
163
+ [sourceEntityId]: {
164
+ sourceEntityId,
165
+ sourceName: "order_summary",
166
+ physicalTableName: "order_summary",
167
+ connectionName: "duckdb",
168
+ dataAsOf: opts.dataAsOf,
169
+ freshnessWindowSeconds: opts.freshnessWindowSeconds,
170
+ freshnessFallback: opts.freshnessFallback,
171
+ },
172
+ },
173
+ }),
174
+ "utf8",
175
+ );
176
+ return file;
177
+ }
178
+
179
+ async function bind(manifestFile: string): Promise<void> {
180
+ const patchRes = await fetch(url(""), {
181
+ method: "PATCH",
182
+ headers: { "Content-Type": "application/json" },
183
+ body: JSON.stringify({
184
+ name: PACKAGE_NAME,
185
+ manifestLocation: manifestFile,
186
+ }),
187
+ });
188
+ expect(patchRes.status).toBe(200);
189
+ const patched = (await patchRes.json()) as Record<string, unknown>;
190
+ // The bind is recorded regardless of per-query freshness — freshness is
191
+ // evaluated per query, not at bind.
192
+ expect(patched.manifestBindingStatus).toBe("bound");
193
+ expect(patched.manifestEntryCount).toBe(1);
194
+ }
195
+
196
+ async function revertToLive(): Promise<void> {
197
+ await fetch(url(""), {
198
+ method: "PATCH",
199
+ headers: { "Content-Type": "application/json" },
200
+ body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
201
+ });
202
+ await fetch(url("?reload=true"));
203
+ }
204
+
205
+ it(
206
+ "routes a fresh entry to the materialized table",
207
+ async () => {
208
+ const manifestFile = await writeFreshnessManifest({
209
+ dataAsOf: new Date().toISOString(),
210
+ freshnessWindowSeconds: 86_400, // 1 day — comfortably fresh
211
+ freshnessFallback: "live",
212
+ });
213
+ await bind(manifestFile);
214
+
215
+ const routed = await executedSql();
216
+ expect(routed).not.toContain("data/orders.csv");
217
+ expect(routed).toContain("order_summary");
218
+
219
+ await revertToLive();
220
+ },
221
+ { timeout: 120_000 },
222
+ );
223
+
224
+ it(
225
+ "serves live for a stale entry with fallback live",
226
+ async () => {
227
+ const manifestFile = await writeFreshnessManifest({
228
+ // 2h old against a 1h window ⇒ stale.
229
+ dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
230
+ freshnessWindowSeconds: 3600,
231
+ freshnessFallback: "live",
232
+ });
233
+ await bind(manifestFile);
234
+
235
+ const served = await executedSql();
236
+ expect(served).toContain("data/orders.csv");
237
+
238
+ await revertToLive();
239
+ },
240
+ { timeout: 120_000 },
241
+ );
242
+
243
+ it(
244
+ "serves the stale table for a stale entry with fallback stale_ok",
245
+ async () => {
246
+ const manifestFile = await writeFreshnessManifest({
247
+ dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
248
+ freshnessWindowSeconds: 3600,
249
+ freshnessFallback: "stale_ok",
250
+ });
251
+ await bind(manifestFile);
252
+
253
+ const served = await executedSql();
254
+ expect(served).not.toContain("data/orders.csv");
255
+ expect(served).toContain("order_summary");
256
+
257
+ await revertToLive();
258
+ },
259
+ { timeout: 120_000 },
260
+ );
261
+
262
+ it(
263
+ "serves live once a fresh entry crosses its window, with no rebind",
264
+ async () => {
265
+ const windowSeconds = 10;
266
+ const anchor = Date.now();
267
+ const manifestFile = await writeFreshnessManifest({
268
+ dataAsOf: new Date(anchor).toISOString(),
269
+ freshnessWindowSeconds: windowSeconds,
270
+ freshnessFallback: "live",
271
+ });
272
+ await bind(manifestFile);
273
+
274
+ // While inside the window, the served query routes to the table.
275
+ expect(Date.now() - anchor).toBeLessThan(windowSeconds * 1000);
276
+ const fresh = await executedSql();
277
+ expect(fresh).not.toContain("data/orders.csv");
278
+ expect(fresh).toContain("order_summary");
279
+
280
+ // Wait for the window to elapse — no rebind, no reload.
281
+ const waitMs = windowSeconds * 1000 - (Date.now() - anchor) + 1500;
282
+ await new Promise((r) => setTimeout(r, Math.max(waitMs, 0)));
283
+
284
+ // The very next query re-evaluates freshness and now serves live.
285
+ const afterCrossing = await executedSql();
286
+ expect(afterCrossing).toContain("data/orders.csv");
287
+
288
+ await revertToLive();
289
+ },
290
+ { timeout: 120_000 },
291
+ );
292
+ });