@malloy-publisher/server 0.0.224 → 0.0.225

package/src/service/package.ts

@@ -521,6 +521,17 @@ export class Package {
       return this.buildPlan;
    }
 
+   /**
+    * The package-level `materialization` config (from malloy-publisher.json),
+    * the least-specific layer for resolving per-source freshness/schedule in the
+    * build plan. Null when the package declares no policy.
+    */
+   public getMaterializationConfig(): NonNullable<
+      ApiPackage["materialization"]
+   > | null {
+      return this.packageMetadata.materialization ?? null;
+   }
+
    public getPackageMetadata(): ApiPackage {
       // Overlay the server-computed fields onto the stored metadata: the
       // explores misconfig warnings (loading is fail-safe — the package still
@@ -706,27 +717,61 @@ export class Package {
    }
 
    /**
-    * Publish-gate for the package-level materialization cron. The package-level
-    * cron (`materialization.schedule`) is replaced outright at Phase B by
-    * `materialization.freshness.window` (persistence.md §9.4), so any declared
-    * cron is rejected. The `sharing=private` carve-out — a cron may legitimately
-    * own a package's own private artifacts — arrives whole with Phase A; until
-    * `sharing=private` is declarable there is no private artifact for a cron to
-    * own, so the B→A rule is to reject any declared cron outright
-    * (persistence-phase-b-design.md §3.4). One actionable message pointing at
-    * `materialization.freshness.window` (empty when no cron is declared). Strict
-    * at publish (package.controller), warn-only at load/reload (loadViaWorker) —
-    * same split as explores.
+    * Publish-gate for materialization crons (Phase A carve-out, persistence.md
+    * §9.4). A cron is the power tier of artifact-anchored scheduling and is
+    * valid only over `sharing="private"` artifacts; a cron on a shared (or unset
+    * ⇒ shared) artifact is rejected, pointing at `freshness.window` as the
+    * shared-tier replacement. Two grains, both enforced off the resolved build
+    * plan (per-source `sharing`/`schedule` are effective, most-specific-wins):
+    *
+    *  - Per-source: a source's own `schedule` is valid only when that source
+    *    resolves to explicit `sharing="private"`.
+    *  - Package: `materialization.schedule` governs every persist source, so it
+    *    is valid only when every governed source resolves to explicit
+    *    `sharing="private"` (a mixed/shared/unset package cannot carry it).
+    *
+    * Strict at publish (package.controller), warn-only at load/reload
+    * (loadViaWorker) — same split as explores.
     */
    public scheduleWarnings(): string[] {
-      const schedule = this.packageMetadata.materialization?.schedule;
-      if (!schedule) return [];
-      return [
-         `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is not accepted: ` +
-            `package-level crons are rejected in Phase B. Declare ` +
-            `'materialization.freshness.window' instead (the control plane schedules ` +
-            `refreshes from it).`,
-      ];
+      const warnings: string[] = [];
+      const sources = this.buildPlan?.sources
+         ? Object.values(this.buildPlan.sources)
+         : [];
+
+      for (const source of sources) {
+         if (source.schedule && source.sharing !== "private") {
+            warnings.push(
+               `#@ persist source "${source.name}" declares a schedule (cron) but ` +
+                  `resolves to sharing="${
+                     source.sharing ?? "shared (unset default)"
+                  }": a per-source cron is valid only for sharing="private". ` +
+                  `Declare 'freshness.window' instead (the control plane schedules ` +
+                  `refreshes from it).`,
+            );
+         }
+      }
+
+      const packageSchedule = this.packageMetadata.materialization?.schedule;
+      if (packageSchedule) {
+         const nonPrivate = sources.filter((s) => s.sharing !== "private");
+         if (sources.length === 0 || nonPrivate.length > 0) {
+            const detail =
+               sources.length === 0
+                  ? "the package declares no persist source for it to govern"
+                  : `sources [${nonPrivate
+                       .map((s) => s.name)
+                       .join(", ")}] resolve to shared/unset`;
+            warnings.push(
+               `materialization.schedule (cron) in ${PACKAGE_MANIFEST_NAME} is valid ` +
+                  `only when every persist source resolves to sharing="private": ` +
+                  `${detail}. Declare 'materialization.freshness.window' instead ` +
+                  `(the control plane schedules refreshes from it).`,
+            );
+         }
+      }
+
+      return warnings;
    }
 
    /** The {@link scheduleWarnings} joined into one string, or "" if none. */

package/src/storage/DatabaseInterface.ts

@@ -116,6 +116,7 @@ export type BuildPlan = components["schemas"]["BuildPlan"];
 export type BuildManifestResult = components["schemas"]["BuildManifest"];
 export type ManifestEntry = components["schemas"]["ManifestEntry"];
 export type BuildInstruction = components["schemas"]["BuildInstruction"];
+export type ManifestReference = components["schemas"]["ManifestReference"];
 export type Realization = components["schemas"]["Realization"];
 
 export interface Materialization {

package/tests/fixtures/persist-multi-level/data/orders.csv

@@ -0,0 +1,5 @@
+category,amount
+electronics,100
+clothing,50
+electronics,200
+clothing,75

package/tests/fixtures/persist-multi-level/multi_level.malloy

@@ -0,0 +1,18 @@
+##! experimental.persistence
+
+source: raw_orders is duckdb.table('data/orders.csv')
+
+// Upstream persist source: an aggregate over the base CSV.
+#@ persist name="orders_base"
+source: orders_base is raw_orders -> {
+  group_by: category
+  aggregate: total_amount is amount.sum()
+}
+
+// Downstream persist source: references the persist upstream `orders_base`.
+// A per-unit build of `orders_rollup` alone must REFERENCE orders_base's
+// materialized table (via referenceManifest) rather than recompute it live.
+#@ persist name="orders_rollup"
+source: orders_rollup is orders_base -> {
+  aggregate: category_count is count()
+}

package/tests/fixtures/persist-multi-level/publisher.json

@@ -0,0 +1,5 @@
+{
+  "name": "persist-multi-level",
+  "version": "1.0.0",
+  "description": "DuckDB two-level persist DAG fixture (orders_base -> orders_rollup)"
+}

package/tests/integration/materialization/reference_manifest.integration.spec.ts

@@ -0,0 +1,251 @@
+/// <reference types="bun-types" />
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const PROJECT_NAME = "test-project-refmanifest";
+const PACKAGE_NAME = "persist-multi-level";
+const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
+
+const TERMINAL_STATUSES = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
+
+/**
+ * WI-1 (Phase D): the orchestrated build seeds its build Manifest from
+ * `referenceManifest` and honors `strictUpstreams`. Proven end-to-end over a
+ * two-level persist DAG (`orders_base` -> `orders_rollup`):
+ *
+ *  - Build the downstream `orders_rollup` ALONE with a `referenceManifest`
+ *    pointing at the already-materialized `orders_base` table and
+ *    `strictUpstreams=true` -> the build SUCCEEDS, which under strict mode is
+ *    only possible if the upstream reference resolved to the physical table
+ *    (a strict miss would throw).
+ *  - Build `orders_rollup` ALONE under `strictUpstreams=true` with NO
+ *    `referenceManifest` -> the build FAILS loudly (runtime-manifest-strict-miss)
+ *    instead of silently recomputing the upstream live.
+ */
+describe("Materialization reference manifest + strictUpstreams (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+
+      const fixtureDir = path.resolve(
+         __dirname,
+         "../../fixtures/persist-multi-level",
+      );
+      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         const body = await createRes.text();
+         throw new Error(
+            `Failed to create test project (${createRes.status}): ${body}`,
+         );
+      }
+
+      const deadline = Date.now() + 30_000;
+      let pkgReady = false;
+      while (!pkgReady && Date.now() < deadline) {
+         try {
+            const res = await fetch(`${baseUrl}${API}`);
+            if (res.ok) {
+               pkgReady = true;
+               break;
+            }
+         } catch {
+            // not ready yet
+         }
+         await new Promise((r) => setTimeout(r, 500));
+      }
+      if (!pkgReady) {
+         throw new Error("Test package did not become available in time");
+      }
+   });
+
+   afterAll(async () => {
+      if (baseUrl) {
+         try {
+            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort cleanup
+         }
+      }
+      await env?.stop();
+      env = null;
+   });
+
+   function url(p: string): string {
+      return `${baseUrl}${API}${p}`;
+   }
+
+   async function createMaterialization(
+      body: Record<string, unknown> = {},
+   ): Promise<Response> {
+      return fetch(url("/materializations"), {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify(body),
+      });
+   }
+
+   async function pollUntilTerminal(
+      id: string,
+      timeoutMs = 90_000,
+   ): Promise<Record<string, unknown>> {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+         const res = await fetch(url(`/materializations/${id}`));
+         expect(res.status).toBe(200);
+         const data = (await res.json()) as Record<string, unknown>;
+         if (TERMINAL_STATUSES.includes(data.status as string)) return data;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      throw new Error(`Materialization ${id} did not reach a terminal state`);
+   }
+
+   /** Delete a terminal materialization record (optionally dropping its tables). */
+   async function deleteRun(id: string, dropTables = false): Promise<void> {
+      await fetch(
+         url(`/materializations/${id}${dropTables ? "?dropTables=true" : ""}`),
+         { method: "DELETE" },
+      );
+   }
+
+   /** Build one source alone and drive it to a terminal state. */
+   async function buildOneAlone(
+      body: Record<string, unknown>,
+   ): Promise<Record<string, unknown>> {
+      const createRes = await createMaterialization(body);
+      expect(createRes.status).toBe(201);
+      const { id } = (await createRes.json()) as { id: string };
+      const settled = await pollUntilTerminal(id);
+      return { id, ...settled };
+   }
+
+   /** The planned sources keyed by name (off Package.buildPlan). */
+   async function planSourcesByName(): Promise<
+      Record<string, Record<string, unknown>>
+   > {
+      const res = await fetch(url(""));
+      expect(res.status).toBe(200);
+      const pkg = (await res.json()) as Record<string, unknown>;
+      const plan = pkg.buildPlan as Record<string, unknown>;
+      expect(plan).toBeDefined();
+      const sources = plan.sources as Record<string, Record<string, unknown>>;
+      const byName: Record<string, Record<string, unknown>> = {};
+      for (const s of Object.values(sources)) {
+         byName[s.name as string] = s;
+      }
+      return byName;
+   }
+
+   const UPSTREAM_TABLE = "orders_base_built";
+
+   it(
+      "builds a downstream source alone, referencing the upstream's physical table under strict",
+      async () => {
+         const sources = await planSourcesByName();
+         const base = sources["orders_base"];
+         const rollup = sources["orders_rollup"];
+         expect(base).toBeDefined();
+         expect(rollup).toBeDefined();
+
+         // 1) Materialize the upstream `orders_base` into a known physical table.
+         const baseRun = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: base.sourceEntityId,
+                     sourceID: base.sourceID,
+                     materializedTableId: "mt-base",
+                     physicalTableName: UPSTREAM_TABLE,
+                     realization: "COPY",
+                  },
+               ],
+            },
+         });
+         expect(baseRun.status).toBe("MANIFEST_FILE_READY");
+         await deleteRun(baseRun.id as string); // keep the physical table
+
+         // 2) Build the downstream `orders_rollup` ALONE, referencing the
+         // upstream's physical table under strictUpstreams. Success is only
+         // possible if the reference resolved (strict forbids the live fallback).
+         const rollupRun = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: rollup.sourceEntityId,
+                     sourceID: rollup.sourceID,
+                     materializedTableId: "mt-rollup",
+                     physicalTableName: "orders_rollup_built",
+                     realization: "COPY",
+                  },
+               ],
+               referenceManifest: [
+                  {
+                     sourceEntityId: base.sourceEntityId,
+                     physicalTableName: UPSTREAM_TABLE,
+                  },
+               ],
+               strictUpstreams: true,
+            },
+         });
+         expect(rollupRun.status).toBe("MANIFEST_FILE_READY");
+
+         // Cleanup both physical tables and records.
+         await deleteRun(rollupRun.id as string, true);
+         // Drop the upstream table via a fresh no-op build+drop is overkill;
+         // the environment teardown removes the project. Best-effort direct drop
+         // is unnecessary for correctness here.
+      },
+      { timeout: 120_000 },
+   );
+
+   it(
+      "fails loudly when a strict upstream is neither built nor referenced",
+      async () => {
+         const sources = await planSourcesByName();
+         const rollup = sources["orders_rollup"];
+         expect(rollup).toBeDefined();
+
+         // Build `orders_rollup` ALONE under strictUpstreams with NO
+         // referenceManifest for its upstream -> the compile throws
+         // runtime-manifest-strict-miss and the run FAILS.
+         const run = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: rollup.sourceEntityId,
+                     sourceID: rollup.sourceID,
+                     materializedTableId: "mt-rollup-strict",
+                     physicalTableName: "orders_rollup_strict",
+                     realization: "COPY",
+                  },
+               ],
+               strictUpstreams: true,
+            },
+         });
+
+         expect(run.status).toBe("FAILED");
+         expect(String(run.error ?? "")).toContain("manifest");
+
+         await deleteRun(run.id as string, true);
+      },
+      { timeout: 120_000 },
+   );
+});