@malloy-publisher/server 0.0.224 → 0.0.226

package/tests/integration/materialization/reference_manifest.integration.spec.ts

@@ -0,0 +1,251 @@
+/// <reference types="bun-types" />
+
+import { afterAll, beforeAll, describe, expect, it } from "bun:test";
+import path from "path";
+import { fileURLToPath } from "url";
+import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const PROJECT_NAME = "test-project-refmanifest";
+const PACKAGE_NAME = "persist-multi-level";
+const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
+
+const TERMINAL_STATUSES = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
+
+/**
+ * WI-1 (Phase D): the orchestrated build seeds its build Manifest from
+ * `referenceManifest` and honors `strictUpstreams`. Proven end-to-end over a
+ * two-level persist DAG (`orders_base` -> `orders_rollup`):
+ *
+ *  - Build the downstream `orders_rollup` ALONE with a `referenceManifest`
+ *    pointing at the already-materialized `orders_base` table and
+ *    `strictUpstreams=true` -> the build SUCCEEDS, which under strict mode is
+ *    only possible if the upstream reference resolved to the physical table
+ *    (a strict miss would throw).
+ *  - Build `orders_rollup` ALONE under `strictUpstreams=true` with NO
+ *    `referenceManifest` -> the build FAILS loudly (runtime-manifest-strict-miss)
+ *    instead of silently recomputing the upstream live.
+ */
+describe("Materialization reference manifest + strictUpstreams (E2E)", () => {
+   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
+   let baseUrl: string;
+
+   beforeAll(async () => {
+      env = await startRestE2E();
+      baseUrl = env.baseUrl;
+
+      const fixtureDir = path.resolve(
+         __dirname,
+         "../../fixtures/persist-multi-level",
+      );
+      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify({
+            name: PROJECT_NAME,
+            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
+            connections: [],
+         }),
+      });
+      if (!createRes.ok) {
+         const body = await createRes.text();
+         throw new Error(
+            `Failed to create test project (${createRes.status}): ${body}`,
+         );
+      }
+
+      const deadline = Date.now() + 30_000;
+      let pkgReady = false;
+      while (!pkgReady && Date.now() < deadline) {
+         try {
+            const res = await fetch(`${baseUrl}${API}`);
+            if (res.ok) {
+               pkgReady = true;
+               break;
+            }
+         } catch {
+            // not ready yet
+         }
+         await new Promise((r) => setTimeout(r, 500));
+      }
+      if (!pkgReady) {
+         throw new Error("Test package did not become available in time");
+      }
+   });
+
+   afterAll(async () => {
+      if (baseUrl) {
+         try {
+            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
+               method: "DELETE",
+            });
+         } catch {
+            // best-effort cleanup
+         }
+      }
+      await env?.stop();
+      env = null;
+   });
+
+   function url(p: string): string {
+      return `${baseUrl}${API}${p}`;
+   }
+
+   async function createMaterialization(
+      body: Record<string, unknown> = {},
+   ): Promise<Response> {
+      return fetch(url("/materializations"), {
+         method: "POST",
+         headers: { "Content-Type": "application/json" },
+         body: JSON.stringify(body),
+      });
+   }
+
+   async function pollUntilTerminal(
+      id: string,
+      timeoutMs = 90_000,
+   ): Promise<Record<string, unknown>> {
+      const deadline = Date.now() + timeoutMs;
+      while (Date.now() < deadline) {
+         const res = await fetch(url(`/materializations/${id}`));
+         expect(res.status).toBe(200);
+         const data = (await res.json()) as Record<string, unknown>;
+         if (TERMINAL_STATUSES.includes(data.status as string)) return data;
+         await new Promise((r) => setTimeout(r, 250));
+      }
+      throw new Error(`Materialization ${id} did not reach a terminal state`);
+   }
+
+   /** Delete a terminal materialization record (optionally dropping its tables). */
+   async function deleteRun(id: string, dropTables = false): Promise<void> {
+      await fetch(
+         url(`/materializations/${id}${dropTables ? "?dropTables=true" : ""}`),
+         { method: "DELETE" },
+      );
+   }
+
+   /** Build one source alone and drive it to a terminal state. */
+   async function buildOneAlone(
+      body: Record<string, unknown>,
+   ): Promise<Record<string, unknown>> {
+      const createRes = await createMaterialization(body);
+      expect(createRes.status).toBe(201);
+      const { id } = (await createRes.json()) as { id: string };
+      const settled = await pollUntilTerminal(id);
+      return { id, ...settled };
+   }
+
+   /** The planned sources keyed by name (off Package.buildPlan). */
+   async function planSourcesByName(): Promise<
+      Record<string, Record<string, unknown>>
+   > {
+      const res = await fetch(url(""));
+      expect(res.status).toBe(200);
+      const pkg = (await res.json()) as Record<string, unknown>;
+      const plan = pkg.buildPlan as Record<string, unknown>;
+      expect(plan).toBeDefined();
+      const sources = plan.sources as Record<string, Record<string, unknown>>;
+      const byName: Record<string, Record<string, unknown>> = {};
+      for (const s of Object.values(sources)) {
+         byName[s.name as string] = s;
+      }
+      return byName;
+   }
+
+   const UPSTREAM_TABLE = "orders_base_built";
+
+   it(
+      "builds a downstream source alone, referencing the upstream's physical table under strict",
+      async () => {
+         const sources = await planSourcesByName();
+         const base = sources["orders_base"];
+         const rollup = sources["orders_rollup"];
+         expect(base).toBeDefined();
+         expect(rollup).toBeDefined();
+
+         // 1) Materialize the upstream `orders_base` into a known physical table.
+         const baseRun = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: base.sourceEntityId,
+                     sourceID: base.sourceID,
+                     materializedTableId: "mt-base",
+                     physicalTableName: UPSTREAM_TABLE,
+                     realization: "COPY",
+                  },
+               ],
+            },
+         });
+         expect(baseRun.status).toBe("MANIFEST_FILE_READY");
+         await deleteRun(baseRun.id as string); // keep the physical table
+
+         // 2) Build the downstream `orders_rollup` ALONE, referencing the
+         // upstream's physical table under strictUpstreams. Success is only
+         // possible if the reference resolved (strict forbids the live fallback).
+         const rollupRun = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: rollup.sourceEntityId,
+                     sourceID: rollup.sourceID,
+                     materializedTableId: "mt-rollup",
+                     physicalTableName: "orders_rollup_built",
+                     realization: "COPY",
+                  },
+               ],
+               referenceManifest: [
+                  {
+                     sourceEntityId: base.sourceEntityId,
+                     physicalTableName: UPSTREAM_TABLE,
+                  },
+               ],
+               strictUpstreams: true,
+            },
+         });
+         expect(rollupRun.status).toBe("MANIFEST_FILE_READY");
+
+         // Cleanup both physical tables and records.
+         await deleteRun(rollupRun.id as string, true);
+         // Drop the upstream table via a fresh no-op build+drop is overkill;
+         // the environment teardown removes the project. Best-effort direct drop
+         // is unnecessary for correctness here.
+      },
+      { timeout: 120_000 },
+   );
+
+   it(
+      "fails loudly when a strict upstream is neither built nor referenced",
+      async () => {
+         const sources = await planSourcesByName();
+         const rollup = sources["orders_rollup"];
+         expect(rollup).toBeDefined();
+
+         // Build `orders_rollup` ALONE under strictUpstreams with NO
+         // referenceManifest for its upstream -> the compile throws
+         // runtime-manifest-strict-miss and the run FAILS.
+         const run = await buildOneAlone({
+            buildInstructions: {
+               sources: [
+                  {
+                     sourceEntityId: rollup.sourceEntityId,
+                     sourceID: rollup.sourceID,
+                     materializedTableId: "mt-rollup-strict",
+                     physicalTableName: "orders_rollup_strict",
+                     realization: "COPY",
+                  },
+               ],
+               strictUpstreams: true,
+            },
+         });
+
+         expect(run.status).toBe("FAILED");
+         expect(String(run.error ?? "")).toContain("manifest");
+
+         await deleteRun(run.id as string, true);
+      },
+      { timeout: 120_000 },
+   );
+});

package/src/service/materialization_cron_gate.spec.ts

@@ -1,142 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import * as fs from "fs/promises";
-import * as os from "os";
-import * as path from "path";
-
-import { Environment } from "./environment";
-import type { Package } from "./package";
-
-/**
- * The publish gate for the package-level materialization cron. At Phase B the
- * package-level cron (`materialization.schedule`) is replaced outright by
- * `materialization.freshness.window` (persistence.md §9.4), so ANY declared cron
- * is rejected regardless of the sources' sharing. The `sharing=private`
- * carve-out arrives whole with Phase A (persistence-phase-b-design.md §3.4);
- * until then there is no private artifact for a cron to own, so the B→A rule is
- * reject-all. These tests run a real `Environment` + `Package.create` over temp
- * dirs, so they also prove end-to-end that the pinned compiler ACCEPTS
- * `sharing=` / `refresh=` keys in the `#@ persist` annotation and that the
- * values survive to the wire build plan verbatim (unset stays null — never
- * defaulted to "shared").
- */
-describe("materialization cron gate", () => {
-   let rootDir: string;
-   let envPath: string;
-
-   async function loadPackage(model: string, schedule?: string) {
-      const dir = path.join(envPath, "pkg");
-      await fs.mkdir(dir, { recursive: true });
-      await fs.writeFile(
-         path.join(dir, "publisher.json"),
-         JSON.stringify({
-            name: "pkg",
-            description: "fixture",
-            ...(schedule ? { materialization: { schedule } } : {}),
-         }),
-      );
-      await fs.writeFile(path.join(dir, "model.malloy"), model);
-      const env = await Environment.create("testEnv", envPath, []);
-      await env.addPackage("pkg");
-      return env.getPackage("pkg", false);
-   }
-
-   function sourceByName(pkg: Package, name: string) {
-      const sources = pkg.getBuildPlan()?.sources ?? {};
-      const found = Object.values(sources).find((s) => s.name === name);
-      if (!found) throw new Error(`persist source '${name}' not in build plan`);
-      return found;
-   }
-
-   beforeEach(async () => {
-      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-cron-"));
-      envPath = path.join(rootDir, "env");
-      await fs.mkdir(envPath, { recursive: true });
-   });
-
-   afterEach(async () => {
-      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
-   });
-
-   const MIXED_MODEL = `##! experimental.persistence
-
-#@ persist name="priv_table" sharing=private refresh=incremental
-source: priv is duckdb.sql("SELECT 1 as x")
-
-#@ persist name="open_table" sharing=shared
-source: open is duckdb.sql("SELECT 2 as x")
-
-#@ persist name="unset_table"
-source: unspecified is duckdb.sql("SELECT 3 as x")
-`;
-
-   const ALL_PRIVATE_MODEL = `##! experimental.persistence
-
-#@ persist name="a_table" sharing=private
-source: a is duckdb.sql("SELECT 1 as x")
-
-#@ persist name="b_table" sharing=private refresh=full
-source: b is duckdb.sql("SELECT 2 as x")
-`;
-
-   it(
-      "surfaces declared sharing/refresh verbatim on the build plan (null when unset)",
-      async () => {
-         const pkg = await loadPackage(MIXED_MODEL);
-
-         const priv = sourceByName(pkg, "priv");
-         expect(priv.sharing).toBe("private");
-         expect(priv.refresh).toBe("incremental");
-
-         const open = sourceByName(pkg, "open");
-         expect(open.sharing).toBe("shared");
-         expect(open.refresh).toBeNull();
-
-         // Unset must be reported as null — distinguishable from an explicit
-         // "shared" — because the control plane applies the platform default
-         // (unset => shared) itself.
-         const unspecified = sourceByName(pkg, "unspecified");
-         expect(unspecified.sharing).toBeNull();
-         expect(unspecified.refresh).toBeNull();
-      },
-      { timeout: 30000 },
-   );
-
-   it(
-      "rejects any declared cron, pointing at freshness.window",
-      async () => {
-         const pkg = await loadPackage(MIXED_MODEL, "0 6 * * *");
-
-         // One actionable message for the whole package — the B→A rule is
-         // reject-all, not per-source.
-         const warnings = pkg.scheduleWarnings();
-         expect(warnings).toHaveLength(1);
-         const joined = pkg.formatInvalidSchedule();
-         expect(joined).toContain("materialization.schedule");
-         expect(joined).toContain("materialization.freshness.window");
-      },
-      { timeout: 30000 },
-   );
-
-   it(
-      "rejects a declared cron even when every persist source is private",
-      async () => {
-         // The sharing=private carve-out arrives with Phase A; during B→A even
-         // an all-private package's cron is rejected outright.
-         const pkg = await loadPackage(ALL_PRIVATE_MODEL, "0 6 * * *");
-         expect(pkg.scheduleWarnings()).toHaveLength(1);
-         expect(pkg.formatInvalidSchedule()).toContain(
-            "materialization.freshness.window",
-         );
-      },
-      { timeout: 30000 },
-   );
-
-   it(
-      "is inert when no cron is declared, whatever the sources' sharing",
-      async () => {
-         const pkg = await loadPackage(MIXED_MODEL);
-         expect(pkg.scheduleWarnings()).toEqual([]);
-      },
-      { timeout: 30000 },
-   );
-});