@malloy-publisher/server 0.0.220 → 0.0.222

package/src/service/materialization_schedule_surface.spec.ts

@@ -53,6 +53,30 @@ describe("materialization schedule surfacing", () => {
          const pkg = await env.getPackage("pkg", false);
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: "0 6 * * *",
+            freshness: null,
+         });
+      },
+      { timeout: 20000 },
+   );
+
+   it(
+      "surfaces the manifest's materialization.freshness on load",
+      async () => {
+         const env = await Environment.create("testEnv", envPath, []);
+         await writePackageDir({
+            materialization: {
+               freshness: { window: "24h", fallback: "stale_ok" },
+            },
+         });
+         await env.addPackage("pkg");
+
+         // Freshness rides the same channel as schedule: present-and-verbatim
+         // when declared, null when unset — the control plane owns the
+         // scheduling/gating logic and only needs the values surfaced.
+         const pkg = await env.getPackage("pkg", false);
+         expect(pkg.getPackageMetadata().materialization).toEqual({
+            schedule: null,
+            freshness: { window: "24h", fallback: "stale_ok" },
          });
       },
       { timeout: 20000 },
@@ -71,6 +95,7 @@ describe("materialization schedule surfacing", () => {
          const pkg = await env.getPackage("pkg", false);
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: null,
+            freshness: null,
          });
       },
       { timeout: 20000 },
@@ -91,11 +116,15 @@ describe("materialization schedule surfacing", () => {
             name: "pkg",
             description: "updated",
          });
-         expect(updated.materialization).toEqual({ schedule: "0 6 * * *" });
+         expect(updated.materialization).toEqual({
+            schedule: "0 6 * * *",
+            freshness: null,
+         });
 
          const pkg = await env.getPackage("pkg", false);
          expect(pkg.getPackageMetadata().materialization).toEqual({
             schedule: "0 6 * * *",
+            freshness: null,
          });
       },
       { timeout: 20000 },

package/src/service/materialization_test_fixtures.ts

@@ -81,7 +81,10 @@ export function fakeSource(opts: {
    sql?: string;
    connectionName?: string;
    dialectName?: string;
+   /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
+   annotationFields?: Record<string, string>;
 }): PersistSource {
+   const fields = opts.annotationFields;
    return {
       name: opts.name,
       sourceID: opts.name,
@@ -90,7 +93,20 @@ export function fakeSource(opts: {
       makeBuildId: () => opts.sourceEntityId,
       getSQL: () => opts.sql ?? "SELECT 1",
       annotations: {
-         parseAsTag: () => ({ tag: { text: () => undefined } }),
+         parseAsTag: () =>
+            fields
+               ? {
+                    tag: {
+                       *entries() {
+                          for (const [key, value] of Object.entries(fields)) {
+                             yield [key, { text: () => value }];
+                          }
+                       },
+                    },
+                 }
+               : // No annotation fields: entries() is absent, and
+                 // deriveAnnotationFields degrades to {}.
+                 { tag: { text: () => undefined } },
       },
    } as unknown as PersistSource;
 }

package/src/service/package.ts

@@ -347,6 +347,7 @@ export class Package {
          // absence as a schedule removal. See `parsePackageMaterialization`.
          materialization: outcome.packageMetadata.materialization ?? {
             schedule: null,
+            freshness: null,
          },
       };
 
@@ -450,6 +451,16 @@ export class Package {
             detail: invalidMsg,
          });
       }
+      // Same fail-safe split for the materialization cron gate: an existing
+      // package whose manifest violates the private-only cron rule still loads
+      // (warn), but a publish of it is rejected (see package.controller).
+      const invalidSchedule = pkg.formatInvalidSchedule();
+      if (invalidSchedule) {
+         logger.warn(`Package ${packageName} has an invalid cron schedule`, {
+            packageName,
+            detail: invalidSchedule,
+         });
+      }
       pkg.logEmptyDiscoveryWarnings();
 
       return pkg;
@@ -603,6 +614,41 @@ export class Package {
       return this.exploreWarnings(exploresOverride).join("\n");
    }
 
+   /**
+    * Publish-gate for the package-level materialization cron (the power tier
+    * of artifact-anchored scheduling): a declared `materialization.schedule`
+    * governs every persist source in the package, so it is valid only when
+    * each source in the compiled build plan resolves to an explicit
+    * `sharing=private`. A cron over any shared/unset source would let one
+    * package dictate the refresh cadence of an artifact other packages share —
+    * those packages declare `materialization.freshness.window` instead and the
+    * control plane schedules the shared artifact from the tightest window.
+    * One actionable message per offending source (empty when the cron is
+    * valid or no cron is declared). Strict at publish (package.controller),
+    * warn-only at load/reload (loadViaWorker) — same split as explores.
+    */
+   public scheduleWarnings(): string[] {
+      const schedule = this.packageMetadata.materialization?.schedule;
+      if (!schedule) return [];
+      const sources = Object.values(this.buildPlan?.sources ?? {});
+      return sources
+         .filter((source) => source.sharing !== "private")
+         .map(
+            (source) =>
+               `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires every ` +
+               `persist source to declare '#@ persist ... sharing=private'; source ` +
+               `'${source.name}' resolves to ${
+                  source.sharing ? `'${source.sharing}'` : "unset"
+               }. Declare 'materialization.freshness.window' instead (the control plane ` +
+               `schedules refreshes from it), or mark every persist source sharing=private.`,
+         );
+   }
+
+   /** The {@link scheduleWarnings} joined into one string, or "" if none. */
+   public formatInvalidSchedule(): string {
+      return this.scheduleWarnings().join("\n");
+   }
+
    /**
     * One message per LISTED model whose discovery surface is empty because it
     * is import-only (imports other files, declares/re-exports nothing). Such a

package/src/service/package_manifest.spec.ts

@@ -5,7 +5,7 @@ describe("service/package_manifest", () => {
    describe("parsePackageMaterialization", () => {
       it("extracts a string schedule", () => {
          expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(
-            { schedule: "0 6 * * *" },
+            { schedule: "0 6 * * *", freshness: null },
          );
       });
 
@@ -18,21 +18,59 @@ describe("service/package_manifest", () => {
          expect(
             parsePackageMaterialization({
                schedule: "*/15 * * * *",
-               freshness: { window: "24h" },
+               retries: 3,
             }),
-         ).toEqual({ schedule: "*/15 * * * *" });
+         ).toEqual({ schedule: "*/15 * * * *", freshness: null });
       });
 
       it("degrades a non-string schedule to null", () => {
          expect(parsePackageMaterialization({ schedule: 42 })).toEqual({
             schedule: null,
+            freshness: null,
+         });
+         expect(parsePackageMaterialization({})).toEqual({
+            schedule: null,
+            freshness: null,
          });
-         expect(parsePackageMaterialization({})).toEqual({ schedule: null });
       });
 
       it("returns null for non-object input", () => {
          expect(parsePackageMaterialization("0 6 * * *")).toBeNull();
          expect(parsePackageMaterialization(7)).toBeNull();
       });
+
+      it("extracts a full freshness block verbatim", () => {
+         expect(
+            parsePackageMaterialization({
+               freshness: { window: "24h", fallback: "stale_ok" },
+            }),
+         ).toEqual({
+            schedule: null,
+            freshness: { window: "24h", fallback: "stale_ok" },
+         });
+      });
+
+      it("keeps a partial freshness block (window only)", () => {
+         expect(
+            parsePackageMaterialization({ freshness: { window: "1h" } }),
+         ).toEqual({ schedule: null, freshness: { window: "1h" } });
+      });
+
+      it("drops invalid freshness fields rather than defaulting them", () => {
+         // A bad value is reported as absent — never substituted — so the
+         // control plane sees exactly what was (validly) declared.
+         expect(
+            parsePackageMaterialization({
+               freshness: { window: 24, fallback: "retry" },
+            }),
+         ).toEqual({ schedule: null, freshness: {} });
+      });
+
+      it("degrades a non-object freshness to null", () => {
+         expect(parsePackageMaterialization({ freshness: "24h" })).toEqual({
+            schedule: null,
+            freshness: null,
+         });
+      });
    });
 });

package/src/service/package_manifest.ts

@@ -4,18 +4,61 @@
  * testable in isolation from the package-load worker that consumes it.
  */
 
+const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
+export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
+
+/**
+ * The manifest's `materialization.freshness` block, surfaced verbatim for the
+ * control plane (which owns the scheduling and query-time gating logic).
+ * Fields are kept only when valid — an invalid value is dropped, never
+ * defaulted, so absence on the wire always means "not declared".
+ */
+export interface PackageFreshnessConfig {
+   /** Maximum acceptable staleness, as a duration string (e.g. "24h"). */
+   window?: string;
+   /** Declared query-time behavior when the window is missed. */
+   fallback?: FreshnessFallback;
+}
+
 export interface PackageMaterializationConfig {
    /**
     * 5-field UNIX cron the control plane uses to schedule version-level
-    * re-materialization. Null when absent or not a string.
+    * re-materialization. Null when absent or not a string. Publish-gated:
+    * only valid when every persist source resolves to explicit
+    * `sharing=private` (see Package.scheduleWarnings).
     */
    schedule: string | null;
+   /**
+    * Freshness policy ({ window, fallback }). Null when the manifest declares
+    * none — distinct from an empty object, so the control plane can tell
+    * "no policy" from "policy with no valid fields".
+    */
+   freshness: PackageFreshnessConfig | null;
+}
+
+function parseFreshness(raw: unknown): PackageFreshnessConfig | null {
+   if (!raw || typeof raw !== "object") {
+      return null;
+   }
+   const { window, fallback } = raw as { window?: unknown; fallback?: unknown };
+   const freshness: PackageFreshnessConfig = {};
+   if (typeof window === "string") {
+      freshness.window = window;
+   }
+   if (
+      typeof fallback === "string" &&
+      (FRESHNESS_FALLBACKS as readonly string[]).includes(fallback)
+   ) {
+      freshness.fallback = fallback as FreshnessFallback;
+   }
+   return freshness;
 }
 
 /**
  * Read the manifest's `materialization` object, keeping only recognized fields.
  * Returns null when the block is absent so the API field is null rather than an
- * empty object; a non-string schedule degrades to null.
+ * empty object; a non-string schedule degrades to null, and an absent or
+ * non-object freshness degrades to null.
  */
 export function parsePackageMaterialization(
    raw: unknown,
@@ -23,6 +66,12 @@ export function parsePackageMaterialization(
    if (!raw || typeof raw !== "object") {
       return null;
    }
-   const schedule = (raw as { schedule?: unknown }).schedule;
-   return { schedule: typeof schedule === "string" ? schedule : null };
+   const { schedule, freshness } = raw as {
+      schedule?: unknown;
+      freshness?: unknown;
+   };
+   return {
+      schedule: typeof schedule === "string" ? schedule : null,
+      freshness: parseFreshness(freshness),
+   };
 }