@malloy-publisher/server 0.0.223 → 0.0.225

package/src/service/manifest_loader.ts

@@ -4,7 +4,7 @@ import * as fs from "fs/promises";
 import { fileURLToPath } from "url";
 import { components } from "../api";
 import { logger } from "../logger";
-import { BuildManifest } from "../storage/DatabaseInterface";
+import { FreshnessManifest } from "../storage/DatabaseInterface";
 
 type WireBuildManifest = components["schemas"]["BuildManifest"];
 
@@ -58,14 +58,18 @@ async function readManifestBytes(uri: string): Promise<string> {
 
 /**
  * Fetch and parse the control-plane-computed build manifest at `uri`, returning
- * the Malloy-runtime binding map (`sourceEntityId -> { tableName }`). The wire manifest
- * keys physical tables under `physicalTableName`; the Malloy runtime consumes
- * `tableName`, so we translate here. Entries missing a physical table are
- * skipped. Throws if the URI can't be read or parsed.
+ * the full wire binding map (`sourceEntityId -> { tableName, dataAsOf,
+ * freshnessWindowSeconds, freshnessFallback }`). The wire manifest keys physical
+ * tables under `physicalTableName`; the Malloy runtime consumes `tableName`, so
+ * we translate that field here but otherwise carry the control-plane freshness
+ * fields verbatim so the serve path can gate `age vs window` per query. Entries
+ * missing a physical table are skipped. This is a pure fetch + parse: it does
+ * **not** filter on freshness (that happens per query on the serve path). Throws
+ * if the URI can't be read or parsed.
  */
 export async function fetchManifestEntries(
    uri: string,
-): Promise<BuildManifest["entries"]> {
+): Promise<FreshnessManifest> {
    const raw = await readManifestBytes(uri);
 
    let parsed: WireBuildManifest;
@@ -79,7 +83,7 @@ export async function fetchManifestEntries(
       );
    }
 
-   const entries: BuildManifest["entries"] = {};
+   const entries: FreshnessManifest = {};
    for (const [sourceEntityId, entry] of Object.entries(parsed.entries ?? {})) {
       const physicalTableName = entry?.physicalTableName;
       if (!physicalTableName) {
@@ -89,7 +93,12 @@ export async function fetchManifestEntries(
          });
          continue;
       }
-      entries[sourceEntityId] = { tableName: physicalTableName };
+      entries[sourceEntityId] = {
+         tableName: physicalTableName,
+         dataAsOf: entry.dataAsOf,
+         freshnessWindowSeconds: entry.freshnessWindowSeconds,
+         freshnessFallback: entry.freshnessFallback,
+      };
    }
    return entries;
 }

package/src/service/materialization_cron_gate.spec.ts

@@ -7,14 +7,22 @@ import { Environment } from "./environment";
 import type { Package } from "./package";
 
 /**
- * The publish gate for the package-level materialization cron (§9.4
- * artifact-anchored scheduling): a declared `materialization.schedule` is the
- * power tier and is valid only when every persist source in the compiled
- * build plan resolves to an explicit `sharing=private`. These tests run a real
- * `Environment` + `Package.create` over temp dirs, so they also prove
- * end-to-end that the pinned compiler ACCEPTS `sharing=` / `refresh=` keys in
- * the `#@ persist` annotation and that the values survive to the wire build
- * plan verbatim (unset stays null — never defaulted to "shared").
+ * The publish gate for materialization crons (Phase A carve-out,
+ * persistence.md §9.4). A cron is the power tier of artifact-anchored
+ * scheduling and is valid only over `sharing="private"` artifacts:
+ *
+ *  - a package-level cron (`materialization.schedule`) is valid only when
+ *    EVERY governed persist source resolves to explicit `sharing="private"`;
+ *  - a per-source cron (`#@ persist ... schedule=`) is valid only when THAT
+ *    source resolves to explicit `sharing="private"`;
+ *  - a cron on a shared/unset artifact is rejected, pointing at
+ *    `freshness.window` as the shared-tier replacement.
+ *
+ * These tests run a real `Environment` + `Package.create` over temp dirs, so
+ * they also prove end-to-end that the pinned compiler ACCEPTS `sharing=` /
+ * `refresh=` / `schedule=` / `freshness.*` keys in the `#@ persist` annotation
+ * and that the values survive to the wire build plan verbatim (unset stays null
+ * — never defaulted to "shared").
  */
 describe("materialization cron gate", () => {
    let rootDir: string;
@@ -73,6 +81,20 @@ source: a is duckdb.sql("SELECT 1 as x")
 
 #@ persist name="b_table" sharing=private refresh=full
 source: b is duckdb.sql("SELECT 2 as x")
+`;
+
+   // Per-source crons: valid on the private source, rejected on the shared and
+   // the unset (⇒ shared) sources.
+   const PER_SOURCE_CRON_MODEL = `##! experimental.persistence
+
+#@ persist name="p_table" sharing=private schedule="0 */6 * * *"
+source: p is duckdb.sql("SELECT 1 as x")
+
+#@ persist name="s_table" sharing=shared schedule="0 0 * * *"
+source: s is duckdb.sql("SELECT 2 as x")
+
+#@ persist name="u_table" schedule="0 0 * * *"
+source: u is duckdb.sql("SELECT 3 as x")
 `;
 
    it(
@@ -99,29 +121,51 @@ source: b is duckdb.sql("SELECT 2 as x")
    );
 
    it(
-      "rejects a cron over shared/unset persist sources, pointing at freshness.window",
+      "rejects a package cron on a mixed package, pointing at freshness.window",
       async () => {
+         // Phase A: a package cron governs every persist source, so it is valid
+         // only when all resolve to explicit private. MIXED has a shared and an
+         // unset source, so the package cron is rejected.
          const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
 
          const warnings = pkg.scheduleWarnings();
-         // One actionable message per offending source; the compliant private
-         // source is not flagged.
-         expect(warnings).toHaveLength(2);
+         expect(warnings).toHaveLength(1);
          const joined = pkg.formatInvalidSchedule();
-         expect(joined).toContain("'open' resolves to 'shared'");
-         expect(joined).toContain("'unspecified' resolves to unset");
-         expect(joined).not.toContain("'priv'");
+         expect(joined).toContain("materialization.schedule");
          expect(joined).toContain("materialization.freshness.window");
       },
       { timeout: 30000 },
    );
 
    it(
-      "allows a cron when every persist source is explicitly private",
+      "accepts a package cron when every persist source is private",
       async () => {
+         // Phase A carve-out: an all-private package legitimately carries a
+         // package-level cron (the power tier for its own private artifacts).
          const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
          expect(pkg.scheduleWarnings()).toEqual([]);
-         expect(pkg.formatInvalidSchedule()).toBe("");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a per-source cron unless that source resolves to private",
+      async () => {
+         // No package cron here — only per-source `schedule=` declarations. The
+         // private source's cron is accepted; the shared and unset sources' are
+         // rejected, each pointing at freshness.window.
+         const pkg = await loadPackage(PER_SOURCE_CRON_MODEL);
+
+         const warnings = pkg.scheduleWarnings();
+         expect(warnings).toHaveLength(2);
+         const joined = warnings.join("\n");
+         expect(joined).toContain('"s"');
+         expect(joined).toContain('"u"');
+         expect(joined).not.toContain('"p"');
+         expect(joined).toContain("freshness.window");
+
+         // The resolved per-source schedule is surfaced verbatim on the plan.
+         expect(sourceByName(pkg, "p").schedule).toBe("0 */6 * * *");
       },
       { timeout: 30000 },
    );

package/src/service/materialization_service.spec.ts

@@ -921,6 +921,10 @@ describe("runBuild (branch behavior)", () => {
             sourceNames: string[] | undefined;
             forceRefresh: boolean;
             buildInstructions: BuildInstruction[] | undefined;
+            referenceManifest?:
+               | { sourceEntityId: string; physicalTableName: string }[]
+               | undefined;
+            strictUpstreams?: boolean | undefined;
          },
          signal: AbortSignal,
       ) => Promise<void>;
@@ -972,6 +976,44 @@ describe("runBuild (branch behavior)", () => {
       expect(svc.autoLoadManifest.called).toBe(false);
    });
 
+   it("orchestrated: seeds the build from referenceManifest and honors strictUpstreams", async () => {
+      const svc = stubEngine();
+      const instructions = [makeInstruction()];
+
+      await svc.runBuild(
+         "mat-1",
+         "my-env",
+         "pkg",
+         {
+            sourceNames: undefined,
+            forceRefresh: false,
+            buildInstructions: instructions,
+            referenceManifest: [
+               { sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
+            ],
+            strictUpstreams: true,
+         },
+         new AbortController().signal,
+      );
+
+      // The reference manifest becomes the seed (carried) entries so a
+      // downstream build resolves its upstream to the existing physical table.
+      expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
+         "up-1": {
+            sourceEntityId: "up-1",
+            physicalTableName: "upstream_tbl",
+         },
+      });
+      // strictUpstreams flows through as the strict flag (5th arg).
+      expect(svc.executeInstructedBuild.firstCall.args[4]).toBe(true);
+      // Reused upstreams are counted as carried, not built.
+      expect(svc.commitManifest.firstCall.args[2]).toMatchObject({
+         mode: "orchestrated",
+         sourcesBuilt: 1,
+         sourcesReused: 1,
+      });
+   });
+
    it("auto-run: derives instructions and auto-loads the manifest", async () => {
       const svc = stubEngine();
 

package/src/service/materialization_service.ts

@@ -23,10 +23,12 @@ import {
    BuildManifest,
    BuildManifestResult,
    BuildPlan,
+   FreshnessManifest,
    Materialization,
    MaterializationStatus,
    MaterializationUpdate,
    ManifestEntry,
+   ManifestReference,
    ResourceRepository,
 } from "../storage/DatabaseInterface";
 import { DuplicateActiveMaterializationError } from "../storage/duckdb/MaterializationRepository";
@@ -215,6 +217,8 @@ export class MaterializationService {
          forceRefresh?: boolean;
          sourceNames?: string[];
          buildInstructions?: BuildInstruction[];
+         referenceManifest?: ManifestReference[];
+         strictUpstreams?: boolean;
       } = {},
    ): Promise<Materialization> {
       const environmentId = await this.resolveEnvironmentId(environmentName);
@@ -274,6 +278,8 @@ export class MaterializationService {
                sourceNames: options.sourceNames,
                forceRefresh,
                buildInstructions,
+               referenceManifest: options.referenceManifest,
+               strictUpstreams: options.strictUpstreams,
             },
             signal,
          ),
@@ -298,6 +304,8 @@ export class MaterializationService {
          sourceNames: string[] | undefined;
          forceRefresh: boolean;
          buildInstructions: BuildInstruction[] | undefined;
+         referenceManifest: ManifestReference[] | undefined;
+         strictUpstreams: boolean | undefined;
       },
       signal: AbortSignal,
    ): Promise<void> {
@@ -332,7 +340,12 @@ export class MaterializationService {
          let carried: Record<string, ManifestEntry>;
          if (orchestrated) {
             instructions = opts.buildInstructions!;
-            carried = {};
+            // Seed the build Manifest with the caller-supplied upstream
+            // references so a downstream source's persist upstream (built in a
+            // prior run, not rebuilt here) resolves to its existing physical
+            // table instead of recomputing live. The reference key is the
+            // compiler's manifest-lookup sourceEntityId (see ManifestReference).
+            carried = this.referenceManifestToEntries(opts.referenceManifest);
          } else {
             // Skip-if-unchanged: reuse tables from the most recent successful
             // manifest for sources whose sourceEntityId is unchanged, unless
@@ -356,6 +369,7 @@ export class MaterializationService {
             instructions,
             carried,
             signal,
+            opts.strictUpstreams ?? false,
          );
 
          const sourcesBuilt = instructions.length;
@@ -449,6 +463,26 @@ export class MaterializationService {
       return { instructions, carried };
    }
 
+   /**
+    * Project the caller-supplied upstream reference manifest into the seed
+    * entry map `executeInstructedBuild` consumes. Each reference is keyed by the
+    * compiler's manifest-lookup sourceEntityId and carries only the physical
+    * table name — enough for the build Manifest to resolve a downstream persist
+    * reference to the existing table without rebuilding the upstream.
+    */
+   private referenceManifestToEntries(
+      referenceManifest: ManifestReference[] | undefined,
+   ): Record<string, ManifestEntry> {
+      const entries: Record<string, ManifestEntry> = {};
+      for (const ref of referenceManifest ?? []) {
+         entries[ref.sourceEntityId] = {
+            sourceEntityId: ref.sourceEntityId,
+            physicalTableName: ref.physicalTableName,
+         };
+      }
+      return entries;
+   }
+
    /**
     * Entries of the most recent successful (MANIFEST_FILE_READY) materialization
     * for this package, used for skip-if-unchanged. Excludes the in-flight run.
@@ -480,12 +514,16 @@ export class MaterializationService {
       environment: {
          reloadAllModelsForPackage(
             packageName: string,
-            manifest: BuildManifest["entries"],
+            manifest: FreshnessManifest,
          ): Promise<void>;
       },
       packageName: string,
       entries: Record<string, ManifestEntry>,
    ): Promise<void> {
+      // The post-build auto-load binds tableName-only entries: the control plane
+      // stamps freshness (dataAsOf/window/fallback) on the wire manifest it
+      // distributes, not on this in-memory post-build load, so these sources are
+      // bound un-gated (always serve the freshly-built table).
       const manifestEntries: BuildManifest["entries"] = {};
       for (const [sourceEntityId, entry] of Object.entries(entries)) {
          if (entry.physicalTableName) {
@@ -560,6 +598,7 @@ export class MaterializationService {
       instructions: BuildInstruction[],
       seedEntries: Record<string, ManifestEntry>,
       signal: AbortSignal,
+      strict = false,
    ): Promise<Record<string, ManifestEntry>> {
       const { graphs, sources, connectionDigests, connections } = compiled;
 
@@ -582,8 +621,12 @@ export class MaterializationService {
 
       // Accumulates physical names as sources are built so downstream sources
       // resolve their upstream references to the freshly-assigned tables. Seed
-      // it with carried-forward entries so reused upstreams resolve too.
+      // it with carried-forward entries so reused upstreams resolve too. In
+      // strict mode, an upstream persist reference that is neither built here
+      // nor seeded fails the compile (runtime-manifest-strict-miss) instead of
+      // silently recomputing live.
       const manifest = new Manifest();
+      manifest.strict = strict;
       const entries: Record<string, ManifestEntry> = {};
       for (const [sourceEntityId, entry] of Object.entries(seedEntries)) {
          if (entry.physicalTableName) {

package/src/service/materialization_test_fixtures.ts

@@ -75,6 +75,41 @@ export function makeInstruction(
  * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
  * `#@ persist name=` annotation reader, defaulted to "unset").
  */
+/** A freshness/schedule layer for the fake source's `#@` or `##` tag. */
+interface FakeFreshnessSchedule {
+   freshness?: { window?: string; fallback?: string };
+   schedule?: string;
+}
+
+/**
+ * Build a fake Malloy `Tag` supporting both readers the build plan uses:
+ * `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
+ * and the path-based `text(...at)` (dotted `freshness.window` / `schedule`, for
+ * resolveFreshnessSchedule).
+ */
+function fakeTag(
+   fields: Record<string, string> | undefined,
+   fs: FakeFreshnessSchedule | undefined,
+) {
+   return {
+      *entries() {
+         for (const [key, value] of Object.entries(fields ?? {})) {
+            yield [key, { text: () => value }];
+         }
+      },
+      text(...at: string[]): string | undefined {
+         if (at.length === 1) {
+            if (at[0] === "schedule") return fs?.schedule;
+            return fields?.[at[0]];
+         }
+         if (at.length === 2 && at[0] === "freshness") {
+            return fs?.freshness?.[at[1] as "window" | "fallback"];
+         }
+         return undefined;
+      },
+   };
+}
+
 export function fakeSource(opts: {
    name: string;
    sourceEntityId: string;
@@ -83,6 +118,10 @@ export function fakeSource(opts: {
    dialectName?: string;
    /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
    annotationFields?: Record<string, string>;
+   /** Source-level (`#@`) freshness/schedule (dotted keys). */
+   freshnessSchedule?: FakeFreshnessSchedule;
+   /** Model-file-level (`##`) freshness/schedule default. */
+   modelFreshnessSchedule?: FakeFreshnessSchedule;
 }): PersistSource {
    const fields = opts.annotationFields;
    return {
@@ -93,20 +132,14 @@ export function fakeSource(opts: {
       makeBuildId: () => opts.sourceEntityId,
       getSQL: () => opts.sql ?? "SELECT 1",
       annotations: {
-         parseAsTag: () =>
-            fields
-               ? {
-                    tag: {
-                       *entries() {
-                          for (const [key, value] of Object.entries(fields)) {
-                             yield [key, { text: () => value }];
-                          }
-                       },
-                    },
-                 }
-               : // No annotation fields: entries() is absent, and
-                 // deriveAnnotationFields degrades to {}.
-                 { tag: { text: () => undefined } },
+         parseAsTag: () => ({
+            tag: fakeTag(fields, opts.freshnessSchedule),
+         }),
+      },
+      modelAnnotations: {
+         parseAsTag: () => ({
+            tag: fakeTag(undefined, opts.modelFreshnessSchedule),
+         }),
       },
    } as unknown as PersistSource;
 }

package/src/service/model.ts

@@ -164,6 +164,14 @@ export class Model {
       exploresDeclared: boolean;
       isQueryEntryPoint: boolean;
    } = { mode: "all", exploresDeclared: false, isQueryEntryPoint: true };
+   /** Per-query freshness resolver, pushed down by the owning Package (see
+    *  Package.wireFreshnessResolvers). Returns the freshness-filtered build
+    *  manifest for the serve path — threaded into Malloy's per-query
+    *  `buildManifest` override so a persist source only routes to its
+    *  materialized table while within its declared freshness window. Undefined
+    *  (or returning undefined) means no override: the runtime-baked manifest
+    *  applies, which serves live when unbound. */
+   private freshnessResolver?: () => BuildManifest["entries"] | undefined;
    private meter = publisherMeter();
    private queryExecutionHistogram = this.meter.createHistogram(
       "malloy_model_query_duration",
@@ -743,6 +751,29 @@ export class Model {
       this.discoveryCurationEnabled = enabled;
    }
 
+   /**
+    * Set by the owning Package (see Package.wireFreshnessResolvers). Supplies
+    * the freshness-filtered build manifest the serve path threads into Malloy's
+    * per-query `buildManifest` override so stale persist sources fall back per
+    * their declared policy. See {@link resolveFreshBuildManifest}.
+    */
+   public setFreshnessResolver(
+      resolver: () => BuildManifest["entries"] | undefined,
+   ): void {
+      this.freshnessResolver = resolver;
+   }
+
+   /**
+    * The freshness-filtered build manifest for this query, or undefined when the
+    * package is unbound / has no resolver (⇒ no per-query override; the runtime
+    * serves live). Evaluated per call so a table that crosses its window while
+    * the package stays loaded is gated on the very next query.
+    */
+   private resolveFreshBuildManifest(): BuildManifest | undefined {
+      const entries = this.freshnessResolver?.();
+      return entries ? { entries, strict: false } : undefined;
+   }
+
    public getSources(): ApiSource[] | undefined {
       return this.curateForDiscovery(this.sources);
    }
@@ -1273,8 +1304,14 @@ export class Model {
 
       const maxRows = getMaxQueryRows();
       const maxBytes = getMaxResponseBytes();
+      // Per-query freshness gate (persistence.md §9.3): resolve the
+      // freshness-filtered manifest once and thread it into both the prepare
+      // (for the row limit) and the run so a stale persist source falls back per
+      // its declared policy — and prep/run agree on the same substitution.
+      const buildManifest = this.resolveFreshBuildManifest();
       const rowLimit = resolveModelQueryRowLimit(
-         (await runnable.getPreparedResult({ givens })).resultExplore.limit,
+         (await runnable.getPreparedResult({ givens, buildManifest }))
+            .resultExplore.limit,
          { defaultLimit: getDefaultQueryRowLimit(), maxRows },
       );
       const endTime = performance.now();
@@ -1282,7 +1319,12 @@ export class Model {
 
       let queryResults;
       try {
-         queryResults = await runnable.run({ rowLimit, givens, abortSignal });
+         queryResults = await runnable.run({
+            rowLimit,
+            givens,
+            abortSignal,
+            buildManifest,
+         });
       } catch (error) {
          // Record error metrics
          const errorEndTime = performance.now();
@@ -1521,9 +1563,16 @@ export class Model {
 
             const cellMaxRows = getMaxQueryRows();
             const cellMaxBytes = getMaxResponseBytes();
+            // Per-query freshness gate (see getQueryResults): the same
+            // freshness-filtered manifest gates notebook-cell queries.
+            const buildManifest = this.resolveFreshBuildManifest();
             const rowLimit = resolveModelQueryRowLimit(
-               (await runnableToExecute.getPreparedResult({ givens }))
-                  .resultExplore.limit,
+               (
+                  await runnableToExecute.getPreparedResult({
+                     givens,
+                     buildManifest,
+                  })
+               ).resultExplore.limit,
                {
                   defaultLimit: getDefaultQueryRowLimit(),
                   maxRows: cellMaxRows,
@@ -1533,6 +1582,7 @@ export class Model {
                rowLimit,
                givens,
                abortSignal,
+               buildManifest,
             });
             const query = (await runnableToExecute.getPreparedQuery())._query;
             queryName = (query as NamedQueryDef).as || query.name;