@malloy-publisher/server 0.0.223 → 0.0.224

package/src/service/package.ts

@@ -30,9 +30,14 @@ import {
 import { formatDuration, logger } from "../logger";
 import { recordBuildPlanComputeDuration } from "../materialization_metrics";
 import { assertSafeEnvironmentPath, safeJoinUnderRoot } from "../path_safety";
-import { BuildManifest, BuildPlan } from "../storage/DatabaseInterface";
+import {
+   BuildManifest,
+   BuildPlan,
+   FreshnessManifest,
+} from "../storage/DatabaseInterface";
 import { errMessage, ignoreDotfiles } from "../utils";
 import { computePackageBuildPlan } from "./build_plan";
+import { filterFreshManifest } from "./freshness";
 import { Model } from "./model";
 import { assertPersistNamesQuoted } from "./persist_annotation_validation";
 
@@ -52,6 +57,22 @@ type PackageConnectionInput =
    | Map<string, Connection>
    | (() => MalloyConfig);
 
+/**
+ * Project the full wire entries down to the Malloy-runtime binding map
+ * (`sourceEntityId -> { tableName }`) used to hydrate models. The freshness
+ * fields are dropped here — they gate the serve path per query, not model
+ * hydration.
+ */
+function toTableNameManifest(
+   entries: FreshnessManifest,
+): BuildManifest["entries"] {
+   const out: BuildManifest["entries"] = {};
+   for (const [sourceEntityId, entry] of Object.entries(entries)) {
+      out[sourceEntityId] = { tableName: entry.tableName };
+   }
+   return out;
+}
+
 export class Package {
    private environmentName: string;
    private packageName: string;
@@ -67,6 +88,22 @@ export class Package {
    // /status (via getPackageMetadata) so the control plane can confirm a worker
    // actually bound the distributed manifest instead of inferring it from logs.
    private buildManifestEntries: BuildManifest["entries"] | undefined;
+   // The full wire entries retained at bind (sourceEntityId -> { tableName,
+   // dataAsOf, freshnessWindowSeconds, freshnessFallback }). Unlike
+   // {@link buildManifestEntries} (the tableName-only projection baked into the
+   // hydration runtime and reused by /compile), this keeps the control-plane
+   // freshness fields so the serve path can re-evaluate `age vs window` per
+   // query. Undefined when the package is serving live (unbound).
+   private freshnessEntries: FreshnessManifest | undefined;
+   // Memoized freshness-filtered manifest for the serve path. Almost all queries
+   // in a window share the same included set, so this is recomputed only when a
+   // retained entry actually crosses its window (`validUntil`, the next
+   // staleSince) or when a rebind replaces the entries (cleared in
+   // recordManifestBinding). validUntil === Infinity means no included entry has
+   // an evaluable window, so the result is stable until the next rebind.
+   private freshManifestCache:
+      | { manifest: BuildManifest["entries"]; validUntil: number }
+      | undefined;
    private manifestBindingStatus: "unbound" | "bound" | "live_fallback" =
       "unbound";
    private manifestEntryCount = 0;
@@ -419,6 +456,10 @@ export class Package {
          malloyConfig,
       );
       pkg.renderTagWarnings = renderTagWarnings;
+      // Install the per-query freshness resolver on the freshly-built models.
+      // At create time no manifest is bound yet, so the resolver returns
+      // undefined (serve live) until a subsequent bindManifest → reloadAllModels.
+      pkg.wireFreshnessResolvers();
 
       // Compute the persist build plan off the live (unbound) models, before the
       // caller binds any configured manifest, so the surfaced plan reflects the
@@ -523,6 +564,51 @@ export class Package {
       return this.buildManifestEntries;
    }
 
+   /**
+    * The freshness-filtered build manifest for the serve path, evaluated at
+    * `now`. Persistence.md §9.3 Phase B: a query on a `#@ persist` source may
+    * use its materialized table only while the table is within its declared
+    * freshness window; otherwise it falls back per the entry's `fallback`
+    * (`live`/`fail` drop the entry → serve live; `stale_ok` keeps it → serve the
+    * stale table). Un-gated entries (no window) always route to the table.
+    *
+    * Undefined when the package is serving live (unbound) — callers then apply
+    * no per-query override and the runtime serves live.
+    *
+    * Memoized: recomputed only when a retained entry crosses its window
+    * (monotonic fresh→stale, so a single `validUntil` deadline suffices) or when
+    * a rebind replaces the entries. Cheap O(1) hit on the common path.
+    */
+   public getFreshBuildManifest(
+      now: number = Date.now(),
+   ): BuildManifest["entries"] | undefined {
+      if (!this.freshnessEntries) return undefined;
+      if (this.freshManifestCache && now < this.freshManifestCache.validUntil) {
+         return this.freshManifestCache.manifest;
+      }
+      const { manifest, nextStaleSince } = filterFreshManifest(
+         this.freshnessEntries,
+         new Date(now),
+      );
+      this.freshManifestCache = {
+         manifest,
+         validUntil: nextStaleSince ?? Infinity,
+      };
+      return manifest;
+   }
+
+   /**
+    * Install the per-query freshness resolver on every owned model so the serve
+    * path (Model.getQueryResults / executeNotebookCell) applies the
+    * freshness-filtered manifest as a per-query Malloy `buildManifest` override.
+    * Idempotent; called after (re)building the model set.
+    */
+   private wireFreshnessResolvers(): void {
+      for (const model of this.models.values()) {
+         model.setFreshnessResolver(() => this.getFreshBuildManifest());
+      }
+   }
+
    /**
     * Record the URI whose manifest is currently bound to the served models. May
     * differ from `manifestLocation` after an in-memory auto-load following a
@@ -545,13 +631,18 @@ export class Package {
     * observability (/status). An empty map means the manifest was cleared, so
     * the package reverts to live (unbound).
     */
-   private recordManifestBinding(
-      buildManifest: BuildManifest["entries"],
-   ): void {
-      const count = Object.keys(buildManifest).length;
-      this.buildManifestEntries = count > 0 ? buildManifest : undefined;
+   private recordManifestBinding(entries: FreshnessManifest): void {
+      const count = Object.keys(entries).length;
+      // Full wire entries (with freshness) drive the per-query serve-path gate;
+      // the tableName-only projection is what /compile and /status consume.
+      this.freshnessEntries = count > 0 ? entries : undefined;
+      this.buildManifestEntries =
+         count > 0 ? toTableNameManifest(entries) : undefined;
       this.manifestEntryCount = count;
       this.manifestBindingStatus = count > 0 ? "bound" : "unbound";
+      // A rebind replaces the entries, so any memoized freshness-filtered
+      // manifest is stale — drop it so the next query recomputes.
+      this.freshManifestCache = undefined;
       if (count === 0) {
          this.boundManifestUri = null;
       }
@@ -615,33 +706,27 @@ export class Package {
    }
 
    /**
-    * Publish-gate for the package-level materialization cron (the power tier
-    * of artifact-anchored scheduling): a declared `materialization.schedule`
-    * governs every persist source in the package, so it is valid only when
-    * each source in the compiled build plan resolves to an explicit
-    * `sharing=private`. A cron over any shared/unset source would let one
-    * package dictate the refresh cadence of an artifact other packages share —
-    * those packages declare `materialization.freshness.window` instead and the
-    * control plane schedules the shared artifact from the tightest window.
-    * One actionable message per offending source (empty when the cron is
-    * valid or no cron is declared). Strict at publish (package.controller),
-    * warn-only at load/reload (loadViaWorker) — same split as explores.
+    * Publish-gate for the package-level materialization cron. The package-level
+    * cron (`materialization.schedule`) is replaced outright at Phase B by
+    * `materialization.freshness.window` (persistence.md §9.4), so any declared
+    * cron is rejected. The `sharing=private` carve-out — a cron may legitimately
+    * own a package's own private artifacts — arrives whole with Phase A; until
+    * `sharing=private` is declarable there is no private artifact for a cron to
+    * own, so the B→A rule is to reject any declared cron outright
+    * (persistence-phase-b-design.md §3.4). One actionable message pointing at
+    * `materialization.freshness.window` (empty when no cron is declared). Strict
+    * at publish (package.controller), warn-only at load/reload (loadViaWorker) —
+    * same split as explores.
     */
    public scheduleWarnings(): string[] {
       const schedule = this.packageMetadata.materialization?.schedule;
       if (!schedule) return [];
-      const sources = Object.values(this.buildPlan?.sources ?? {});
-      return sources
-         .filter((source) => source.sharing !== "private")
-         .map(
-            (source) =>
-               `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` +
-               `persist source to declare '#@ persist ... sharing=private'; source ` +
-               `'${source.name}' resolves to ${
-                  source.sharing ? `'${source.sharing}'` : "unset"
-               }. Declare 'materialization.freshness.window' instead (the control plane ` +
-               `schedules refreshes from it), or mark every persist source sharing=private.`,
-         );
+      return [
+         `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is not accepted: ` +
+            `package-level crons are rejected in Phase B. Declare ` +
+            `'materialization.freshness.window' instead (the control plane schedules ` +
+            `refreshes from it).`,
+      ];
    }
 
    /** The {@link scheduleWarnings} joined into one string, or "" if none. */
@@ -728,9 +813,11 @@ export class Package {
     * timeout, pool shutting down) propagate as `ServiceUnavailableError`
     * — the caller (manifest service) decides how to retry.
     */
-   public async reloadAllModels(
-      buildManifest: BuildManifest["entries"],
-   ): Promise<void> {
+   public async reloadAllModels(entries: FreshnessManifest): Promise<void> {
+      // Models are hydrated against the tableName-only projection; the freshness
+      // fields gate the serve path per query (via getFreshBuildManifest), not
+      // model hydration.
+      const buildManifest = toTableNameManifest(entries);
       const modelPaths = Array.from(this.models.keys());
       logger.info("Reloading all models with build manifest", {
          packageName: this.packageName,
@@ -837,7 +924,13 @@ export class Package {
       this.applyQueryBoundaryToModels();
       // Remember what we just bound so /compile can route identically and
       // /status can report the binding. An empty map reverts to live (unbound).
-      this.recordManifestBinding(buildManifest);
+      // Retains the full freshness entries so the serve-path gate can evaluate
+      // them per query.
+      this.recordManifestBinding(entries);
+      // Install the per-query freshness resolver on the rebuilt model set so the
+      // serve path applies the freshness-filtered manifest as a per-query
+      // override.
+      this.wireFreshnessResolvers();
       // Re-run the fail-safe warning against the refreshed model set: an edit
       // to publisher.json that introduces a bad entry should surface in the
       // logs on reload too, not only at initial load (loadViaWorker).

package/src/storage/DatabaseInterface.ts

@@ -157,3 +157,26 @@ export interface BuildManifest {
    entries: Record<string, BuildManifestEntry>;
    strict?: boolean;
 }
+
+/**
+ * A wire manifest entry carrying the control-plane freshness fields alongside
+ * the physical `tableName`. This is what the publisher retains at bind so the
+ * serve path can re-evaluate `age vs window` per query (see
+ * {@link ../service/freshness}). A `{ tableName }`-only value is structurally
+ * assignable here (the freshness fields are optional), so callers that only
+ * know a table name — e.g. the post-build auto-load — bind un-gated entries.
+ *
+ *  - `dataAsOf`   the artifact's data-as-of instant (snapshot start); the age
+ *                 anchor. `age = now - dataAsOf`. Absent ⇒ never stale.
+ *  - `freshnessWindowSeconds`  the version's effective window. Absent ⇒ un-gated.
+ *  - `freshnessFallback`       per-version read-time policy when stale.
+ */
+export interface FreshnessAwareManifestEntry {
+   tableName: string;
+   dataAsOf?: string;
+   freshnessWindowSeconds?: number;
+   freshnessFallback?: "live" | "stale_ok" | "fail";
+}
+
+/** Full (unfiltered) manifest map keyed by sourceEntityId. */
+export type FreshnessManifest = Record<string, FreshnessAwareManifestEntry>;

package/tests/integration/materialization/freshness_gate.integration.spec.ts

@@ -0,0 +1,292 @@
+/// <reference types="bun-types" />
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import * as fsp from "fs/promises";
+import os from "os";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const PROJECT_NAME = "freshness-gate-project";
+const PACKAGE_NAME = "persist-test";
+const MODEL_PATH = "persist_test.malloy";
+const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
+
+/**
+ * Covers the per-query freshness gate (persistence.md §9.3, §14 Phase B): a
+ * query on a `#@ persist` source uses its materialized table only while the
+ * table is within its declared freshness window; otherwise it falls back per the
+ * entry's declared `fallback`.
+ *
+ *  - fresh (or un-gated)      → routes to the materialized table
+ *  - stale + fallback live    → serves live SQL (entry dropped)
+ *  - stale + fallback stale_ok→ serves the stale table
+ *  - a fresh entry that crosses its window → serves live on the NEXT query,
+ *    with no rebind (proves per-query re-evaluation, not bind-time filtering)
+ *
+ * The manifest is bound once physically (auto-run builds the table), then a
+ * manifest URI keyed by the source's real sourceEntityId routes served queries.
+ * `executedSql()` is the routing evidence: it scans `order_summary` when routed
+ * to the table, and `data/orders.csv` when serving live.
+ */
+describe("Per-query freshness gate (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+   let tmpDir: string;
+   let sourceEntityId: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+      tmpDir = await fsp.mkdtemp(path.join(os.tmpdir(), "freshness-gate-"));
+
+      const fixtureDir = path.resolve(__dirname, "../../fixtures/persist-test");
+      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         throw new Error(
+            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
+         );
+      }
+
+      const deadline = Date.now() + 30_000;
+      while (Date.now() < deadline) {
+         const res = await fetch(`${baseUrl}${API}`);
+         if (res.ok) break;
+         await new Promise((r) => setTimeout(r, 500));
+      }
+
+      // Physically build the persist source (auto-run self-assigns
+      // physicalTableName = "order_summary"), then revert to live so the
+      // manifest-URI bind is the only thing that can route the served query.
+      await buildTableThenRevertToLive();
+      sourceEntityId = await orderSummarySourceEntityId();
+   }, 120_000);
+
+   afterAll(async () => {
+      if (baseUrl) {
+         await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
+            method: "DELETE",
+         }).catch(() => {});
+      }
+      await env?.stop();
+      await fsp.rm(tmpDir, { recursive: true, force: true }).catch(() => {});
+      env = null;
+   });
+
+   function url(p: string): string {
+      return `${baseUrl}${API}${p}`;
+   }
+
+   const ROUTING_QUERY = "run: order_summary -> { aggregate: c is count() }";
+
+   /** The SQL the served query actually compiled to (routing evidence). */
+   async function executedSql(): Promise<string> {
+      const res = await fetch(`${baseUrl}${API}/models/${MODEL_PATH}/query`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({ query: ROUTING_QUERY }),
+      });
+      expect(res.status).toBe(200);
+      const body = (await res.json()) as { result: string };
+      return (JSON.parse(body.result) as { sql: string }).sql;
+   }
+
+   async function orderSummarySourceEntityId(): Promise<string> {
+      const res = await fetch(url(""));
+      expect(res.status).toBe(200);
+      const pkg = (await res.json()) as {
+         buildPlan?: { sources?: Record<string, { sourceEntityId?: string }> };
+      };
+      const sources = pkg.buildPlan?.sources ?? {};
+      const id = Object.values(sources)[0]?.sourceEntityId;
+      expect(typeof id).toBe("string");
+      return id as string;
+   }
+
+   async function pollUntilTerminal(
+      id: string,
+      timeoutMs = 90_000,
+   ): Promise<Record<string, unknown>> {
+      const terminal = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+         const res = await fetch(url(`/materializations/${id}`));
+         expect(res.status).toBe(200);
+         const data = (await res.json()) as Record<string, unknown>;
+         if (terminal.includes(data.status as string)) return data;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      throw new Error(`Materialization ${id} did not reach a terminal state`);
+   }
+
+   async function buildTableThenRevertToLive(): Promise<void> {
+      const createRes = await fetch(url("/materializations"), {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({}),
+      });
+      expect(createRes.status).toBe(201);
+      const { id } = (await createRes.json()) as { id: string };
+      const built = await pollUntilTerminal(id);
+      expect(built.status).toBe("MANIFEST_FILE_READY");
+      await fetch(url("?reload=true"));
+      await fetch(url(`/materializations/${id}`), { method: "DELETE" });
+   }
+
+   /** Write a CP-shaped manifest carrying the freshness fields. */
+   async function writeFreshnessManifest(opts: {
+      dataAsOf: string;
+      freshnessWindowSeconds: number;
+      freshnessFallback: "live" | "stale_ok" | "fail";
+   }): Promise<string> {
+      const file = path.join(
+         tmpDir,
+         `freshness-${Date.now()}-${Math.random()}.json`,
+      );
+      await fsp.writeFile(
+         file,
+         JSON.stringify({
+            builtAt: new Date().toISOString(),
+            strict: false,
+            entries: {
+               [sourceEntityId]: {
+                  sourceEntityId,
+                  sourceName: "order_summary",
+                  physicalTableName: "order_summary",
+                  connectionName: "duckdb",
+                  dataAsOf: opts.dataAsOf,
+                  freshnessWindowSeconds: opts.freshnessWindowSeconds,
+                  freshnessFallback: opts.freshnessFallback,
+               },
+            },
+         }),
+         "utf8",
+      );
+      return file;
+   }
+
+   async function bind(manifestFile: string): Promise<void> {
+      const patchRes = await fetch(url(""), {
+         method: "PATCH",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PACKAGE_NAME,
+            manifestLocation: manifestFile,
+         }),
+      });
+      expect(patchRes.status).toBe(200);
+      const patched = (await patchRes.json()) as Record<string, unknown>;
+      // The bind is recorded regardless of per-query freshness — freshness is
+      // evaluated per query, not at bind.
+      expect(patched.manifestBindingStatus).toBe("bound");
+      expect(patched.manifestEntryCount).toBe(1);
+   }
+
+   async function revertToLive(): Promise<void> {
+      await fetch(url(""), {
+         method: "PATCH",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({ name: PACKAGE_NAME, manifestLocation: null }),
+      });
+      await fetch(url("?reload=true"));
+   }
+
+   it(
+      "routes a fresh entry to the materialized table",
+      async () => {
+         const manifestFile = await writeFreshnessManifest({
+            dataAsOf: new Date().toISOString(),
+            freshnessWindowSeconds: 86_400, // 1 day — comfortably fresh
+            freshnessFallback: "live",
+         });
+         await bind(manifestFile);
+
+         const routed = await executedSql();
+         expect(routed).not.toContain("data/orders.csv");
+         expect(routed).toContain("order_summary");
+
+         await revertToLive();
+      },
+      { timeout: 120_000 },
+   );
+
+   it(
+      "serves live for a stale entry with fallback live",
+      async () => {
+         const manifestFile = await writeFreshnessManifest({
+            // 2h old against a 1h window ⇒ stale.
+            dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
+            freshnessWindowSeconds: 3600,
+            freshnessFallback: "live",
+         });
+         await bind(manifestFile);
+
+         const served = await executedSql();
+         expect(served).toContain("data/orders.csv");
+
+         await revertToLive();
+      },
+      { timeout: 120_000 },
+   );
+
+   it(
+      "serves the stale table for a stale entry with fallback stale_ok",
+      async () => {
+         const manifestFile = await writeFreshnessManifest({
+            dataAsOf: new Date(Date.now() - 7200_000).toISOString(),
+            freshnessWindowSeconds: 3600,
+            freshnessFallback: "stale_ok",
+         });
+         await bind(manifestFile);
+
+         const served = await executedSql();
+         expect(served).not.toContain("data/orders.csv");
+         expect(served).toContain("order_summary");
+
+         await revertToLive();
+      },
+      { timeout: 120_000 },
+   );
+
+   it(
+      "serves live once a fresh entry crosses its window, with no rebind",
+      async () => {
+         const windowSeconds = 10;
+         const anchor = Date.now();
+         const manifestFile = await writeFreshnessManifest({
+            dataAsOf: new Date(anchor).toISOString(),
+            freshnessWindowSeconds: windowSeconds,
+            freshnessFallback: "live",
+         });
+         await bind(manifestFile);
+
+         // While inside the window, the served query routes to the table.
+         expect(Date.now() - anchor).toBeLessThan(windowSeconds * 1000);
+         const fresh = await executedSql();
+         expect(fresh).not.toContain("data/orders.csv");
+         expect(fresh).toContain("order_summary");
+
+         // Wait for the window to elapse — no rebind, no reload.
+         const waitMs = windowSeconds * 1000 - (Date.now() - anchor) + 1500;
+         await new Promise((r) => setTimeout(r, Math.max(waitMs, 0)));
+
+         // The very next query re-evaluates freshness and now serves live.
+         const afterCrossing = await executedSql();
+         expect(afterCrossing).toContain("data/orders.csv");
+
+         await revertToLive();
+      },
+      { timeout: 120_000 },
+   );
+});