@malloy-publisher/server 0.0.224 → 0.0.226

package/src/service/package.ts

@@ -386,6 +386,10 @@ export class Package {
             schedule: null,
             freshness: null,
          },
+         // Package-level persist scope mode, applied uniformly to every persist
+         // source/index. Defaults to "package" (cross-version reuse) when the
+         // manifest omits it.
+         scope: outcome.packageMetadata.scope ?? "package",
       };
 
       // Build live `Model`s from worker output. Any per-model compile
@@ -492,15 +496,19 @@ export class Package {
             detail: invalidMsg,
          });
       }
-      // Same fail-safe split for the materialization cron gate: an existing
-      // package whose manifest violates the private-only cron rule still loads
-      // (warn), but a publish of it is rejected (see package.controller).
-      const invalidSchedule = pkg.formatInvalidSchedule();
-      if (invalidSchedule) {
-         logger.warn(`Package ${packageName} has an invalid cron schedule`, {
-            packageName,
-            detail: invalidSchedule,
-         });
+      // Same fail-safe split for the persistence-policy gate: an existing
+      // package whose manifest violates the scope/schedule/freshness rules
+      // still loads (warn), but a publish of it is rejected (see
+      // package.controller).
+      const invalidPolicy = pkg.formatInvalidPersistencePolicy();
+      if (invalidPolicy) {
+         logger.warn(
+            `Package ${packageName} has an invalid persistence policy`,
+            {
+               packageName,
+               detail: invalidPolicy,
+            },
+         );
       }
       pkg.logEmptyDiscoveryWarnings();
 
@@ -521,6 +529,17 @@ export class Package {
       return this.buildPlan;
    }
 
+   /**
+    * The package-level `materialization` config (from malloy-publisher.json),
+    * the least-specific layer for resolving per-source freshness/schedule in the
+    * build plan. Null when the package declares no policy.
+    */
+   public getMaterializationConfig(): NonNullable<
+      ApiPackage["materialization"]
+   > | null {
+      return this.packageMetadata.materialization ?? null;
+   }
+
    public getPackageMetadata(): ApiPackage {
       // Overlay the server-computed fields onto the stored metadata: the
       // explores misconfig warnings (loading is fail-safe — the package still
@@ -706,32 +725,97 @@ export class Package {
    }
 
    /**
-    * Publish-gate for the package-level materialization cron. The package-level
-    * cron (`materialization.schedule`) is replaced outright at Phase B by
-    * `materialization.freshness.window` (persistence.md §9.4), so any declared
-    * cron is rejected. The `sharing=private` carve-out — a cron may legitimately
-    * own a package's own private artifacts — arrives whole with Phase A; until
-    * `sharing=private` is declarable there is no private artifact for a cron to
-    * own, so the B→A rule is to reject any declared cron outright
-    * (persistence-phase-b-design.md §3.4). One actionable message pointing at
-    * `materialization.freshness.window` (empty when no cron is declared). Strict
-    * at publish (package.controller), warn-only at load/reload (loadViaWorker) —
-    * same split as explores.
+    * Publish-gate for the package's Malloy Persistence policy (persistence.md
+    * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`)
+    * and a materialization cron is package-root-only and version-scope-only.
+    * The rules, read off the resolved build plan + manifest:
+    *
+    *  1. Per-source `#@ persist ... sharing=`/`schedule=` are retired — declaring
+    *     either on a source is an error (scope is package-level; a schedule is
+    *     package-root-only). Detected via the raw `annotationFields`.
+    *  2. `materialization.schedule` is legal only when `scope: version` (a
+    *     package-scoped lineage is reused across versions, so a single
+    *     per-version cadence is meaningless).
+    *  3. `materialization.schedule` and freshness (package `materialization.freshness`
+    *     or any source `freshness`) are mutually exclusive — declare either the
+    *     power tier or the objective tier, never both.
+    *
+    * The cross-package dependency rule (a `scope: version` scheduled package may
+    * not depend on a package-scoped upstream) is not enforced here: scope is
+    * uniform within a package, so an intra-package dependency can never violate
+    * it, and the publisher's build plan is intra-package only — a cross-package
+    * upstream's scope is not resolvable from the single `Package` served. It is
+    * deferred to the scheduler slice (R1–R3), which resolves cross-package scope.
+    *
+    * Strict at publish (package.controller), warn-only at load/reload
+    * (loadViaWorker) — same split as explores.
     */
-   public scheduleWarnings(): string[] {
-      const schedule = this.packageMetadata.materialization?.schedule;
-      if (!schedule) return [];
-      return [
-         `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is not accepted: ` +
-            `package-level crons are rejected in Phase B. Declare ` +
-            `'materialization.freshness.window' instead (the control plane schedules ` +
-            `refreshes from it).`,
-      ];
+   public persistencePolicyWarnings(): string[] {
+      const warnings: string[] = [];
+      const sources = this.buildPlan?.sources
+         ? Object.values(this.buildPlan.sources)
+         : [];
+      const materialization = this.packageMetadata.materialization;
+      const packageSchedule = materialization?.schedule ?? null;
+      const packageFreshness = materialization?.freshness ?? null;
+      const scope = this.packageMetadata.scope ?? "package";
+
+      // Rule 1: per-source sharing/schedule are no longer part of the model.
+      for (const source of sources) {
+         const fields = source.annotationFields ?? {};
+         if (fields.sharing !== undefined) {
+            warnings.push(
+               `#@ persist source "${source.name}" declares sharing=... which is ` +
+                  `no longer supported: scope is a single package-level mode. Set ` +
+                  `the root-level "scope": "version" | "package" in ` +
+                  `${PACKAGE_MANIFEST_NAME} instead.`,
+            );
+         }
+         if (fields.schedule !== undefined) {
+            warnings.push(
+               `#@ persist source "${source.name}" declares schedule=... which is ` +
+                  `no longer supported: a schedule is package-root-only. Declare ` +
+                  `"materialization.schedule" at the ${PACKAGE_MANIFEST_NAME} root ` +
+                  `(requires "scope": "version") instead.`,
+            );
+         }
+      }
+
+      // Rule 2: a package-level cron is legal only with scope: version.
+      if (packageSchedule && scope !== "version") {
+         warnings.push(
+            `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} requires ` +
+               `"scope": "version": a package-scoped lineage is reused across ` +
+               `versions, so a single per-version cadence is meaningless. Set ` +
+               `"scope": "version", or remove the schedule.`,
+         );
+      }
+
+      // Rule 3: schedule and freshness are mutually exclusive.
+      if (packageSchedule) {
+         const freshnessDeclared =
+            !!(
+               packageFreshness &&
+               (packageFreshness.window || packageFreshness.fallback)
+            ) || sources.some((s) => s.freshness != null);
+         if (freshnessDeclared) {
+            warnings.push(
+               `materialization.schedule and freshness are mutually exclusive in ` +
+                  `${PACKAGE_MANIFEST_NAME}: declare either a schedule (power tier) ` +
+                  `or freshness (objective tier), never both.`,
+            );
+         }
+      }
+
+      return warnings;
    }
 
-   /** The {@link scheduleWarnings} joined into one string, or "" if none. */
-   public formatInvalidSchedule(): string {
-      return this.scheduleWarnings().join("\n");
+   /**
+    * The {@link persistencePolicyWarnings} joined into one string, or "" if the
+    * package's persistence policy is valid.
+    */
+   public formatInvalidPersistencePolicy(): string {
+      return this.persistencePolicyWarnings().join("\n");
    }
 
    /**

package/src/service/package_manifest.spec.ts

@@ -1,7 +1,28 @@
 import { describe, expect, it } from "bun:test";
-import { parsePackageMaterialization } from "./package_manifest";
+import {
+   parsePackageMaterialization,
+   parsePackageScope,
+} from "./package_manifest";
 
 describe("service/package_manifest", () => {
+   describe("parsePackageScope", () => {
+      it("defaults to 'package' when absent", () => {
+         expect(parsePackageScope(undefined)).toBe("package");
+         expect(parsePackageScope(null)).toBe("package");
+      });
+
+      it("accepts the two valid modes verbatim", () => {
+         expect(parsePackageScope("version")).toBe("version");
+         expect(parsePackageScope("package")).toBe("package");
+      });
+
+      it("throws on any other value (no silent default)", () => {
+         expect(() => parsePackageScope("shared")).toThrow(/scope/i);
+         expect(() => parsePackageScope("")).toThrow(/scope/i);
+         expect(() => parsePackageScope(7)).toThrow(/scope/i);
+      });
+   });
+
    describe("parsePackageMaterialization", () => {
       it("extracts a string schedule", () => {
          expect(parsePackageMaterialization({ schedule: "0 6 * * *" })).toEqual(

package/src/service/package_manifest.ts

@@ -7,6 +7,35 @@
 const FRESHNESS_FALLBACKS = ["live", "stale_ok", "fail"] as const;
 export type FreshnessFallback = (typeof FRESHNESS_FALLBACKS)[number];
 
+/**
+ * Package-level Malloy Persistence scope mode, declared once at the manifest
+ * root. `package` (the default) = artifacts are reused across the package's
+ * versions when they satisfy freshness; `version` = each artifact is owned by
+ * one published version (the only mode a `materialization.schedule` is legal
+ * in). Applied uniformly to every persist source and index in the package.
+ */
+export const PACKAGE_SCOPES = ["version", "package"] as const;
+export type PackageScope = (typeof PACKAGE_SCOPES)[number];
+
+/**
+ * Read the manifest root's `scope`, defaulting to `"package"` when absent or
+ * null. Any other value is a manifest error: scope is load-bearing (it decides
+ * version-owned vs cross-version reuse), so a typo must fail loudly rather than
+ * silently pick a default. Throws on an invalid value.
+ */
+export function parsePackageScope(raw: unknown): PackageScope {
+   if (raw === undefined || raw === null) {
+      return "package";
+   }
+   if ((PACKAGE_SCOPES as readonly unknown[]).includes(raw)) {
+      return raw as PackageScope;
+   }
+   throw new Error(
+      `Invalid "scope" in the package manifest: ${JSON.stringify(raw)}. ` +
+         `Expected "version" or "package" (default "package").`,
+   );
+}
+
 /**
  * The manifest's `materialization.freshness` block, surfaced verbatim for the
  * control plane (which owns the scheduling and query-time gating logic).

package/src/service/persistence_policy.spec.ts

@@ -0,0 +1,234 @@
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import * as fs from "fs/promises";
+import * as os from "os";
+import * as path from "path";
+
+import { Environment } from "./environment";
+import type { Package } from "./package";
+
+/**
+ * The publish gate for the package's Malloy Persistence policy (persistence.md
+ * §3.1, §9.2–§9.5). Scope is a single package-level mode (`Package.scope`) and a
+ * materialization cron is package-root-only + version-scope-only:
+ *
+ *  - per-source `#@ persist ... sharing=`/`schedule=` are retired (declaring
+ *    either on a source is a manifest error);
+ *  - `materialization.schedule` is legal only when `scope: version`;
+ *  - `materialization.schedule` and freshness (package or source) are mutually
+ *    exclusive.
+ *
+ * These tests run a real `Environment` + `Package.create` over temp dirs, so
+ * they also prove end-to-end that the pinned compiler still ACCEPTS the
+ * `#@ persist` annotation keys (`sharing`/`schedule`/`freshness.*`) at compile
+ * — the rejection is the publisher's manifest-level gate, not a compile error —
+ * and that `scope` is parsed from the manifest root and surfaced on the package.
+ */
+describe("persistence policy gate", () => {
+   let rootDir: string;
+   let envPath: string;
+
+   async function loadPackage(
+      model: string,
+      manifest: Record<string, unknown> = {},
+   ): Promise<Package> {
+      const dir = path.join(envPath, "pkg");
+      await fs.mkdir(dir, { recursive: true });
+      await fs.writeFile(
+         path.join(dir, "publisher.json"),
+         JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
+      );
+      await fs.writeFile(path.join(dir, "model.malloy"), model);
+      const env = await Environment.create("testEnv", envPath, []);
+      await env.addPackage("pkg");
+      return env.getPackage("pkg", false);
+   }
+
+   function sourceByName(pkg: Package, name: string) {
+      const sources = pkg.getBuildPlan()?.sources ?? {};
+      const found = Object.values(sources).find((s) => s.name === name);
+      if (!found) throw new Error(`persist source '${name}' not in build plan`);
+      return found;
+   }
+
+   beforeEach(async () => {
+      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-policy-"));
+      envPath = path.join(rootDir, "env");
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
+   });
+
+   const PLAIN_MODEL = `##! experimental.persistence
+
+#@ persist name="a_table"
+source: a is duckdb.sql("SELECT 1 as x")
+
+#@ persist name="b_table" refresh=full
+source: b is duckdb.sql("SELECT 2 as x")
+`;
+
+   const SHARING_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="s_table" sharing=private
+source: s is duckdb.sql("SELECT 1 as x")
+`;
+
+   const SCHEDULE_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="s_table" schedule="0 */6 * * *"
+source: s is duckdb.sql("SELECT 1 as x")
+`;
+
+   const FRESHNESS_SOURCE_MODEL = `##! experimental.persistence
+
+#@ persist name="f_table" freshness.window="1h"
+source: f is duckdb.sql("SELECT 1 as x")
+`;
+
+   // ── scope parsing + surfacing ─────────────────────────────────────────
+
+   it(
+      "defaults scope to 'package' when the manifest omits it",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL);
+         expect(pkg.getPackageMetadata().scope).toBe("package");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "surfaces an explicit scope: version from the manifest root",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, { scope: "version" });
+         expect(pkg.getPackageMetadata().scope).toBe("version");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "fails the load on an invalid scope value",
+      async () => {
+         await expect(
+            loadPackage(PLAIN_MODEL, { scope: "shared" }),
+         ).rejects.toThrow(/scope/i);
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 1: per-source sharing/schedule retired ───────────────────────
+
+   it(
+      "rejects a per-source sharing= annotation, pointing at root scope",
+      async () => {
+         const pkg = await loadPackage(SHARING_SOURCE_MODEL, {
+            scope: "version",
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain('"s"');
+         expect(joined).toContain("sharing");
+         expect(joined).toContain('"scope"');
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a per-source schedule= annotation, pointing at the root schedule",
+      async () => {
+         const pkg = await loadPackage(SCHEDULE_SOURCE_MODEL, {
+            scope: "version",
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain('"s"');
+         expect(joined).toContain("schedule");
+         expect(joined).toContain("materialization.schedule");
+         // The retired field is not emitted on the wire plan.
+         const source = sourceByName(pkg, "s") as Record<string, unknown>;
+         expect(source.schedule).toBeUndefined();
+         expect(source.sharing).toBeUndefined();
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 2: schedule requires scope: version ──────────────────────────
+
+   it(
+      "rejects a package schedule when scope is package (the default)",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            materialization: { schedule: "0 6 * * *" },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("materialization.schedule");
+         expect(joined).toContain('"scope": "version"');
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "accepts a package schedule when scope is version and no freshness is set",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: { schedule: "0 6 * * *" },
+         });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+      },
+      { timeout: 30000 },
+   );
+
+   // ── Rule 3: schedule XOR freshness ────────────────────────────────────
+
+   it(
+      "rejects a schedule declared alongside package-level freshness",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, {
+            scope: "version",
+            materialization: {
+               schedule: "0 6 * * *",
+               freshness: { window: "24h" },
+            },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("mutually exclusive");
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "rejects a schedule declared alongside a source-level freshness",
+      async () => {
+         const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
+            scope: "version",
+            materialization: { schedule: "0 6 * * *" },
+         });
+         const joined = pkg.formatInvalidPersistencePolicy();
+         expect(joined).toContain("mutually exclusive");
+      },
+      { timeout: 30000 },
+   );
+
+   // ── inert when nothing is declared ────────────────────────────────────
+
+   it(
+      "is inert for a plain package with no schedule and no forbidden knobs",
+      async () => {
+         const pkg = await loadPackage(PLAIN_MODEL, { scope: "package" });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+      },
+      { timeout: 30000 },
+   );
+
+   it(
+      "accepts source-level freshness without a schedule in either scope",
+      async () => {
+         const pkg = await loadPackage(FRESHNESS_SOURCE_MODEL, {
+            scope: "package",
+         });
+         expect(pkg.persistencePolicyWarnings()).toEqual([]);
+         expect(sourceByName(pkg, "f").freshness).toEqual({ window: "1h" });
+      },
+      { timeout: 30000 },
+   );
+});

package/src/storage/DatabaseInterface.ts

@@ -116,6 +116,7 @@ export type BuildPlan = components["schemas"]["BuildPlan"];
 export type BuildManifestResult = components["schemas"]["BuildManifest"];
 export type ManifestEntry = components["schemas"]["ManifestEntry"];
 export type BuildInstruction = components["schemas"]["BuildInstruction"];
+export type ManifestReference = components["schemas"]["ManifestReference"];
 export type Realization = components["schemas"]["Realization"];
 
 export interface Materialization {

package/tests/fixtures/persist-multi-level/data/orders.csv

@@ -0,0 +1,5 @@
+category,amount
+electronics,100
+clothing,50
+electronics,200
+clothing,75

package/tests/fixtures/persist-multi-level/multi_level.malloy

@@ -0,0 +1,18 @@
+##! experimental.persistence
+
+source: raw_orders is duckdb.table('data/orders.csv')
+
+// Upstream persist source: an aggregate over the base CSV.
+#@ persist name="orders_base"
+source: orders_base is raw_orders -> {
+  group_by: category
+  aggregate: total_amount is amount.sum()
+}
+
+// Downstream persist source: references the persist upstream `orders_base`.
+// A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
+// materialized table (via referenceManifest) rather than recompute it live.
+#@ persist name="orders_rollup"
+source: orders_rollup is orders_base -> {
+  aggregate: category_count is count()
+}

package/tests/fixtures/persist-multi-level/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "persist-multi-level",
+  "version": "1.0.0",
+  "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
+}