@malloy-publisher/server 0.0.225 → 0.0.226

package/src/package_load/package_load_pool.ts

@@ -81,7 +81,10 @@ import { fileURLToPath, pathToFileURL } from "url";
 
 import { ModelCompilationError } from "../errors";
 import { logger } from "../logger";
-import type { PackageMaterializationConfig } from "../service/package_manifest";
+import type {
+   PackageMaterializationConfig,
+   PackageScope,
+} from "../service/package_manifest";
 import type {
    ConnectionMetadataRequest,
    ConnectionMetadataResponse,
@@ -235,6 +238,7 @@ export interface LoadPackageOutcome {
       queryableSources?: "declared" | "all";
       manifestLocation?: string | null;
       materialization?: PackageMaterializationConfig | null;
+      scope?: PackageScope;
    };
    models: Array<
       Omit<SerializedModel, "modelDef" | "sourceInfos"> & {

package/src/package_load/package_load_worker.ts

@@ -85,7 +85,9 @@ import { validateAuthorizeProbes } from "../service/authorize";
 import { type FilterDefinition } from "../service/filter";
 import {
    PackageMaterializationConfig,
+   PackageScope,
    parsePackageMaterialization,
+   parsePackageScope,
 } from "../service/package_manifest";
 import {
    extractQueriesFromModelDef,
@@ -387,6 +389,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
    queryableSources?: "declared" | "all";
    manifestLocation?: string | null;
    materialization?: PackageMaterializationConfig | null;
+   scope?: PackageScope;
 }> {
    const manifestPath = path.join(packagePath, PACKAGE_MANIFEST_NAME);
    const contents = await fs.promises.readFile(manifestPath, "utf8");
@@ -397,6 +400,7 @@ async function readPackageMetadata(packagePath: string): Promise<{
       queryableSources?: unknown;
       manifestLocation?: unknown;
       materialization?: unknown;
+      scope?: unknown;
    };
    return {
       name: parsed.name,
@@ -416,6 +420,9 @@ async function readPackageMetadata(packagePath: string): Promise<{
       // Package-level Malloy Persistence policy; surfaced to the control plane,
       // which owns scheduling. Only `schedule` is read today.
       materialization: parsePackageMaterialization(parsed.materialization),
+      // Package-level persist scope mode; defaults to "package". An invalid
+      // value throws here and fails the load (scope is load-bearing).
+      scope: parsePackageScope(parsed.scope),
    };
 }
 

package/src/package_load/protocol.ts

@@ -65,7 +65,10 @@
  */
 
 import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
-import type { PackageMaterializationConfig } from "../service/package_manifest";
+import type {
+   PackageMaterializationConfig,
+   PackageScope,
+} from "../service/package_manifest";
 
 // ──────────────────────────────────────────────────────────────────────
 // Direction: main ──▶ worker (load-package job)
@@ -181,6 +184,7 @@ export interface LoadPackageResult {
       queryableSources?: "declared" | "all";
       manifestLocation?: string | null;
       materialization?: PackageMaterializationConfig | null;
+      scope?: PackageScope;
    };
    models: SerializedModel[];
    /** Wall-clock ms inside the worker for the full package load. */

package/src/service/build_plan.spec.ts

@@ -10,7 +10,7 @@ import {
    deriveBuildPlan,
    flattenDependsOn,
    iterGraphSources,
-   resolveFreshnessSchedule,
+   resolveFreshness,
    resolvePackageConnections,
 } from "./build_plan";
 import { fakeSource } from "./materialization_test_fixtures";
@@ -213,10 +213,10 @@ describe("deriveBuildPlan", () => {
       });
    });
 
-   it("reports declared sharing/refresh verbatim and null when unset", () => {
-      // The control plane distinguishes unset from an explicit `shared` (it
-      // applies the platform default itself), so the publisher must report
-      // the declared value verbatim — never substitute the default.
+   it("reports declared refresh verbatim (null when unset) and does not emit sharing/schedule", () => {
+      // `refresh` is a metadata pass-through; `sharing`/`schedule` were retired
+      // from the contract and must not be emitted as typed fields (they stay in
+      // the raw annotationFields for the publish-time validator to detect).
       const declared = fakeSource({
          name: "declared",
          sourceEntityId: "bid-d",
@@ -243,17 +243,22 @@ describe("deriveBuildPlan", () => {
          { duckdb: "dig" },
       );
 
-      expect(plan.sources["declared@m"].sharing).toBe("private");
       expect(plan.sources["declared@m"].refresh).toBe("incremental");
-      // The raw annotation map still carries every field alongside the typed
-      // projections.
+      // Retired typed fields are absent from the wire projection.
+      expect(
+         (plan.sources["declared@m"] as Record<string, unknown>).sharing,
+      ).toBeUndefined();
+      expect(
+         (plan.sources["declared@m"] as Record<string, unknown>).schedule,
+      ).toBeUndefined();
+      // The raw annotation map still carries every field (so the validator can
+      // reject a source-level sharing/schedule at publish).
       expect(plan.sources["declared@m"].annotationFields).toEqual({
          name: "d_table",
          sharing: "private",
          refresh: "incremental",
       });
-      // Unset is null — not "shared" — on the wire.
-      expect(plan.sources["unset@m"].sharing).toBeNull();
+      // Unset refresh is null on the wire.
       expect(plan.sources["unset@m"].refresh).toBeNull();
    });
 
@@ -297,71 +302,48 @@ describe("deriveBuildPlan", () => {
    });
 });
 
-describe("resolveFreshnessSchedule", () => {
-   it("reports source-level freshness + schedule verbatim", () => {
+describe("resolveFreshness", () => {
+   it("reports source-level freshness verbatim", () => {
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
          freshnessSchedule: {
             freshness: { window: "1h", fallback: "stale_ok" },
-            schedule: "0 */6 * * *",
          },
       });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: { window: "1h", fallback: "stale_ok" },
-         schedule: "0 */6 * * *",
+      expect(resolveFreshness(source, null)).toEqual({
+         window: "1h",
+         fallback: "stale_ok",
       });
    });
 
-   it("returns null for both when unset at every level", () => {
+   it("returns null when unset at every level", () => {
       const source = fakeSource({ name: "s", sourceEntityId: "bid" });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: null,
-         schedule: null,
-      });
+      expect(resolveFreshness(source, null)).toBeNull();
    });
 
    it("falls back to model-file then package per field (most-specific-wins)", () => {
-      // freshness.window from source, freshness.fallback from model-file. The
-      // package-level cron is NOT inherited as the source's effective schedule;
-      // schedule here comes only from the source's own model-file default.
+      // freshness.window from source, freshness.fallback from model-file.
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
          freshnessSchedule: { freshness: { window: "1h" } },
-         modelFreshnessSchedule: {
-            freshness: { fallback: "fail" },
-            schedule: "0 3 * * *",
-         },
+         modelFreshnessSchedule: { freshness: { fallback: "fail" } },
       });
       const pkg = {
-         schedule: "0 0 * * *",
+         schedule: null,
          freshness: { window: "24h", fallback: "live" as const },
       };
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "1h", fallback: "fail" },
-         schedule: "0 3 * * *",
-      });
-   });
-
-   it("does not inherit the package-level cron as the source's schedule", () => {
-      const source = fakeSource({ name: "s", sourceEntityId: "bid" });
-      const pkg = { schedule: "0 0 * * *", freshness: { window: "24h" } };
-      // Freshness inherits from the package; schedule stays null (the package
-      // cron is gated separately, not folded into the per-source cron).
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "24h" },
-         schedule: null,
+      expect(resolveFreshness(source, pkg)).toEqual({
+         window: "1h",
+         fallback: "fail",
       });
    });
 
    it("inherits the package freshness when the source and model are unset", () => {
       const source = fakeSource({ name: "s", sourceEntityId: "bid" });
       const pkg = { schedule: null, freshness: { window: "24h" } };
-      expect(resolveFreshnessSchedule(source, pkg)).toEqual({
-         freshness: { window: "24h" },
-         schedule: null,
-      });
+      expect(resolveFreshness(source, pkg)).toEqual({ window: "24h" });
    });
 
    it("drops an invalid fallback rather than defaulting it", () => {
@@ -370,23 +352,17 @@ describe("resolveFreshnessSchedule", () => {
          sourceEntityId: "bid",
          freshnessSchedule: { freshness: { window: "1h", fallback: "bogus" } },
       });
-      expect(resolveFreshnessSchedule(source, null)).toEqual({
-         freshness: { window: "1h" },
-         schedule: null,
-      });
+      expect(resolveFreshness(source, null)).toEqual({ window: "1h" });
    });
 });
 
-describe("deriveBuildPlan freshness/schedule", () => {
-   it("projects the resolved per-source freshness + schedule onto the plan", () => {
+describe("deriveBuildPlan freshness", () => {
+   it("projects the resolved per-source freshness onto the plan (no schedule/sharing)", () => {
       const source = fakeSource({
          name: "s",
          sourceEntityId: "bid",
-         annotationFields: { name: "s_table", sharing: "private" },
-         freshnessSchedule: {
-            freshness: { window: "1h" },
-            schedule: "0 */6 * * *",
-         },
+         annotationFields: { name: "s_table" },
+         freshnessSchedule: { freshness: { window: "1h" } },
       });
       const plan = deriveBuildPlan(
          [
@@ -406,12 +382,18 @@ describe("deriveBuildPlan freshness/schedule", () => {
       );
 
       // Source window wins over the package default; package fallback fills the
-      // unset source fallback; source schedule reported verbatim.
+      // unset source fallback.
       expect(plan.sources["s@m"].freshness).toEqual({
          window: "1h",
          fallback: "live",
       });
-      expect(plan.sources["s@m"].schedule).toBe("0 */6 * * *");
+      // Retired fields are not emitted.
+      expect(
+         (plan.sources["s@m"] as Record<string, unknown>).schedule,
+      ).toBeUndefined();
+      expect(
+         (plan.sources["s@m"] as Record<string, unknown>).sharing,
+      ).toBeUndefined();
    });
 });
 

package/src/service/build_plan.ts

@@ -25,11 +25,10 @@ type WirePackageMaterialization =
 const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
 type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
 
-/** One layer's contribution to the resolved freshness/schedule (all optional). */
-interface FreshnessScheduleLayer {
+/** One layer's contribution to the resolved freshness (all optional). */
+interface FreshnessLayer {
    window?: string;
    fallback?: FreshnessFallback;
-   schedule?: string;
 }
 
 /** Minimal path-reader over a Malloy `Tag` (see `@malloydata/malloy-tag`). */
@@ -121,16 +120,17 @@ export function deriveAnnotationFields(
 }
 
 /**
- * Read the freshness/schedule keys (`freshness.window`, `freshness.fallback`,
- * `schedule`) from one Malloy tag layer, keeping only recognized values. These
- * are dotted/nested tag properties, so they are NOT captured by the scalar
- * {@link deriveAnnotationFields} loop and must be read by path here.
+ * Read the freshness keys (`freshness.window`, `freshness.fallback`) from one
+ * Malloy tag layer, keeping only recognized values. These are dotted/nested tag
+ * properties, so they are NOT captured by the scalar {@link deriveAnnotationFields}
+ * loop and must be read by path here. (Per-source `sharing`/`schedule` were
+ * retired — scope is package-level and a schedule is package-root-only — so they
+ * are not resolved here; declaring either on a source is a publish-time manifest
+ * error, enforced in Package.persistencePolicyWarnings.)
  */
-function tagFreshnessScheduleLayer(
-   tag: ReadableTag | undefined,
-): FreshnessScheduleLayer {
+function tagFreshnessLayer(tag: ReadableTag | undefined): FreshnessLayer {
    if (!tag || typeof tag.text !== "function") return {};
-   const layer: FreshnessScheduleLayer = {};
+   const layer: FreshnessLayer = {};
    const window = tag.text("freshness", "window");
    if (typeof window === "string") layer.window = window;
    const fallback = tag.text("freshness", "fallback");
@@ -140,24 +140,15 @@ function tagFreshnessScheduleLayer(
    ) {
       layer.fallback = fallback as FreshnessFallback;
    }
-   const schedule = tag.text("schedule");
-   if (typeof schedule === "string") layer.schedule = schedule;
    return layer;
 }
 
-/**
- * The package-level `materialization` config as a freshness resolution layer.
- * Only `freshness` inherits down to a source: the package-level cron
- * (`materialization.schedule`) is a distinct package-grain concept surfaced on
- * `PackageMaterializationConfig.schedule` and gated on its own, so it is NOT
- * folded into a source's effective per-source cron (doing so would double-count
- * against the package cron gate).
- */
+/** The package-level `materialization.freshness` as a resolution layer. */
 function packageFreshnessLayer(
    cfg: WirePackageMaterialization | null | undefined,
-): Pick<FreshnessScheduleLayer, "window" | "fallback"> {
+): FreshnessLayer {
    if (!cfg) return {};
-   const layer: Pick<FreshnessScheduleLayer, "window" | "fallback"> = {};
+   const layer: FreshnessLayer = {};
    if (cfg.freshness?.window) layer.window = cfg.freshness.window;
    const fallback = cfg.freshness?.fallback;
    if (
@@ -192,36 +183,30 @@ function safeModelTag(source: PersistSource): ReadableTag | undefined {
 }
 
 /**
- * Resolve a source's EFFECTIVE freshness + schedule, reported verbatim (unset
- * at every level stays null so the control plane can distinguish "unset" —
- * apply platform default — from an explicit declaration). Invalid `fallback`
- * values are dropped, never defaulted. Precedence, per field, most-specific-wins:
- *
- *  - `freshness`: source annotation > model-file default > package default.
- *  - `schedule`: source annotation > model-file default (the source's OWN cron).
- *    The package-level cron is a package-grain concept, surfaced and gated
- *    separately, so it is not folded into the per-source effective cron.
+ * Resolve a source's EFFECTIVE freshness with most-specific-wins precedence,
+ * per field: source annotation > model-file default > package default. Reported
+ * verbatim (unset at every level stays null so the control plane can
+ * distinguish "unset" — apply platform default — from an explicit declaration).
+ * Invalid `fallback` values are dropped, never defaulted. Freshness is valid in
+ * both scope modes.
  */
-export function resolveFreshnessSchedule(
+export function resolveFreshness(
    source: PersistSource,
    packageMaterialization: WirePackageMaterialization | null | undefined,
-): { freshness: WireFreshness | null; schedule: string | null } {
-   const sourceLayer = tagFreshnessScheduleLayer(safeSourceTag(source));
-   const modelLayer = tagFreshnessScheduleLayer(safeModelTag(source));
+): WireFreshness | null {
+   const sourceLayer = tagFreshnessLayer(safeSourceTag(source));
+   const modelLayer = tagFreshnessLayer(safeModelTag(source));
    const pkgLayer = packageFreshnessLayer(packageMaterialization);
 
    const window = sourceLayer.window ?? modelLayer.window ?? pkgLayer.window;
    const fallback =
       sourceLayer.fallback ?? modelLayer.fallback ?? pkgLayer.fallback;
-   const schedule = sourceLayer.schedule ?? modelLayer.schedule ?? null;
 
-   let freshness: WireFreshness | null = null;
-   if (window !== undefined || fallback !== undefined) {
-      freshness = {};
-      if (window !== undefined) freshness.window = window;
-      if (fallback !== undefined) freshness.fallback = fallback;
-   }
-   return { freshness, schedule };
+   if (window === undefined && fallback === undefined) return null;
+   const freshness: WireFreshness = {};
+   if (window !== undefined) freshness.window = window;
+   if (fallback !== undefined) freshness.fallback = fallback;
+   return freshness;
 }
 
 /** Flatten Malloy's nested BuildNode.dependsOn into a list of sourceIDs. */
@@ -439,15 +424,14 @@ export function deriveBuildPlan(
    for (const [sourceID, source] of Object.entries(sources)) {
       if (include && !include.has(source.name)) continue;
       const annotationFields = deriveAnnotationFields(source);
-      // EFFECTIVE per-source freshness + schedule, resolved most-specific-wins
+      // EFFECTIVE per-source freshness, resolved most-specific-wins
       // (source > model-file > package) and reported verbatim (null = unset at
-      // every level). Unlike `sharing`/`refresh` these are dotted/nested tag
-      // keys, so they come from resolveFreshnessSchedule rather than the scalar
-      // annotationFields map.
-      const { freshness, schedule } = resolveFreshnessSchedule(
-         source,
-         packageMaterialization,
-      );
+      // every level). Freshness is a dotted/nested tag key, so it comes from
+      // resolveFreshness rather than the scalar annotationFields map, and is
+      // valid in both scope modes. Per-source `sharing`/`schedule` are NOT
+      // emitted (retired from the contract); if a source declares either it is
+      // rejected at publish (Package.persistencePolicyWarnings) — the raw keys
+      // still ride `annotationFields` so the validator can detect them.
       wireSources[sourceID] = {
          name: source.name,
          sourceID: source.sourceID,
@@ -455,16 +439,8 @@ export function deriveBuildPlan(
          dialect: source.dialectName,
          sourceEntityId: computeSourceEntityId(source, connectionDigests),
          sql: source.getSQL(),
-         // Declared `#@ persist` scope/refresh knobs, reported VERBATIM (null
-         // = unset). The control plane applies the platform default (unset =>
-         // shared) itself, so the publisher must never substitute it — unset
-         // has to stay distinguishable from an explicit `shared`. The source's
-         // own annotation is the most-specific declaration layer, and the only
-         // one that exists today.
-         sharing: annotationFields.sharing ?? null,
          refresh: annotationFields.refresh ?? null,
-         freshness,
-         schedule,
+         freshness: resolveFreshness(source, packageMaterialization),
          columns: deriveColumns(source),
          annotationFields,
          modelPath: sourceModelPaths?.[sourceID],

package/src/service/environment.ts

@@ -1347,6 +1347,10 @@ export class Environment {
             // control plane misread the gap as a schedule removal. It is not a
             // PATCH-editable field, so always preserve the existing value.
             materialization: existing.materialization,
+            // Same rationale for the manifest-derived persist `scope`: not
+            // PATCH-editable, so preserve it rather than dropping it to the
+            // default until the next reload.
+            scope: existing.scope,
          });
 
          // Strict-reject, symmetric with the publish path

package/src/service/materialization_test_fixtures.ts

@@ -75,17 +75,16 @@ export function makeInstruction(
  * build internals touch (name/id, deterministic sourceEntityId, SQL, and the
  * `#@ persist name=` annotation reader, defaulted to "unset").
  */
-/** A freshness/schedule layer for the fake source's `#@` or `##` tag. */
+/** A freshness layer for the fake source's `#@` or `##` tag. */
 interface FakeFreshnessSchedule {
    freshness?: { window?: string; fallback?: string };
-   schedule?: string;
 }
 
 /**
  * Build a fake Malloy `Tag` supporting both readers the build plan uses:
  * `entries()` (scalar `#@ persist` key=value pairs, for deriveAnnotationFields)
- * and the path-based `text(...at)` (dotted `freshness.window` / `schedule`, for
- * resolveFreshnessSchedule).
+ * and the path-based `text(...at)` (dotted `freshness.window`, for
+ * resolveFreshness).
  */
 function fakeTag(
    fields: Record<string, string> | undefined,
@@ -99,7 +98,6 @@ function fakeTag(
       },
       text(...at: string[]): string | undefined {
          if (at.length === 1) {
-            if (at[0] === "schedule") return fs?.schedule;
             return fields?.[at[0]];
          }
          if (at.length === 2 && at[0] === "freshness") {
@@ -116,11 +114,11 @@ export function fakeSource(opts: {
    sql?: string;
    connectionName?: string;
    dialectName?: string;
-   /** key=value fields of the `#@ persist` annotation (e.g. sharing). */
+   /** key=value fields of the `#@ persist` annotation (e.g. name, refresh). */
    annotationFields?: Record<string, string>;
-   /** Source-level (`#@`) freshness/schedule (dotted keys). */
+   /** Source-level (`#@`) freshness (dotted keys). */
    freshnessSchedule?: FakeFreshnessSchedule;
-   /** Model-file-level (`##`) freshness/schedule default. */
+   /** Model-file-level (`##`) freshness default. */
    modelFreshnessSchedule?: FakeFreshnessSchedule;
 }): PersistSource {
    const fields = opts.annotationFields;