@malloy-publisher/server 0.0.235 → 0.0.236

package/tests/integration/materialization/reference_manifest.integration.spec.ts

@@ -1,251 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const PROJECT_NAME = "test-project-refmanifest";
-const PACKAGE_NAME = "persist-multi-level";
-const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
-
-const TERMINAL_STATUSES = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
-
-/**
- * WI-1 (Phase D): the orchestrated build seeds its build Manifest from
- * `referenceManifest` and honors `strictUpstreams`. Proven end-to-end over a
- * two-level persist DAG (`orders_base` -> `orders_rollup`):
- *
- *  - Build the downstream `orders_rollup` ALONE with a `referenceManifest`
- *    pointing at the already-materialized `orders_base` table and
- *    `strictUpstreams=true` -> the build SUCCEEDS, which under strict mode is
- *    only possible if the upstream reference resolved to the physical table
- *    (a strict miss would throw).
- *  - Build `orders_rollup` ALONE under `strictUpstreams=true` with NO
- *    `referenceManifest` -> the build FAILS loudly (runtime-manifest-strict-miss)
- *    instead of silently recomputing the upstream live.
- */
-describe("Materialization reference manifest + strictUpstreams (E2E)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-
-      const fixtureDir = path.resolve(
-         __dirname,
-         "../../fixtures/persist-multi-level",
-      );
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PROJECT_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         const body = await createRes.text();
-         throw new Error(
-            `Failed to create test project (${createRes.status}): ${body}`,
-         );
-      }
-
-      const deadline = Date.now() + 30_000;
-      let pkgReady = false;
-      while (!pkgReady && Date.now() < deadline) {
-         try {
-            const res = await fetch(`${baseUrl}${API}`);
-            if (res.ok) {
-               pkgReady = true;
-               break;
-            }
-         } catch {
-            // not ready yet
-         }
-         await new Promise((r) => setTimeout(r, 500));
-      }
-      if (!pkgReady) {
-         throw new Error("Test package did not become available in time");
-      }
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort cleanup
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   function url(p: string): string {
-      return `${baseUrl}${API}${p}`;
-   }
-
-   async function createMaterialization(
-      body: Record<string, unknown> = {},
-   ): Promise<Response> {
-      return fetch(url("/materializations"), {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify(body),
-      });
-   }
-
-   async function pollUntilTerminal(
-      id: string,
-      timeoutMs = 90_000,
-   ): Promise<Record<string, unknown>> {
-      const deadline = Date.now() + timeoutMs;
-      while (Date.now() < deadline) {
-         const res = await fetch(url(`/materializations/${id}`));
-         expect(res.status).toBe(200);
-         const data = (await res.json()) as Record<string, unknown>;
-         if (TERMINAL_STATUSES.includes(data.status as string)) return data;
-         await new Promise((r) => setTimeout(r, 250));
-      }
-      throw new Error(`Materialization ${id} did not reach a terminal state`);
-   }
-
-   /** Delete a terminal materialization record (optionally dropping its tables). */
-   async function deleteRun(id: string, dropTables = false): Promise<void> {
-      await fetch(
-         url(`/materializations/${id}${dropTables ? "?dropTables=true" : ""}`),
-         { method: "DELETE" },
-      );
-   }
-
-   /** Build one source alone and drive it to a terminal state. */
-   async function buildOneAlone(
-      body: Record<string, unknown>,
-   ): Promise<Record<string, unknown>> {
-      const createRes = await createMaterialization(body);
-      expect(createRes.status).toBe(201);
-      const { id } = (await createRes.json()) as { id: string };
-      const settled = await pollUntilTerminal(id);
-      return { id, ...settled };
-   }
-
-   /** The planned sources keyed by name (off Package.buildPlan). */
-   async function planSourcesByName(): Promise<
-      Record<string, Record<string, unknown>>
-   > {
-      const res = await fetch(url(""));
-      expect(res.status).toBe(200);
-      const pkg = (await res.json()) as Record<string, unknown>;
-      const plan = pkg.buildPlan as Record<string, unknown>;
-      expect(plan).toBeDefined();
-      const sources = plan.sources as Record<string, Record<string, unknown>>;
-      const byName: Record<string, Record<string, unknown>> = {};
-      for (const s of Object.values(sources)) {
-         byName[s.name as string] = s;
-      }
-      return byName;
-   }
-
-   const UPSTREAM_TABLE = "orders_base_built";
-
-   it(
-      "builds a downstream source alone, referencing the upstream's physical table under strict",
-      async () => {
-         const sources = await planSourcesByName();
-         const base = sources["orders_base"];
-         const rollup = sources["orders_rollup"];
-         expect(base).toBeDefined();
-         expect(rollup).toBeDefined();
-
-         // 1) Materialize the upstream `orders_base` into a known physical table.
-         const baseRun = await buildOneAlone({
-            buildInstructions: {
-               sources: [
-                  {
-                     sourceEntityId: base.sourceEntityId,
-                     sourceID: base.sourceID,
-                     materializedTableId: "mt-base",
-                     physicalTableName: UPSTREAM_TABLE,
-                     realization: "COPY",
-                  },
-               ],
-            },
-         });
-         expect(baseRun.status).toBe("MANIFEST_FILE_READY");
-         await deleteRun(baseRun.id as string); // keep the physical table
-
-         // 2) Build the downstream `orders_rollup` ALONE, referencing the
-         // upstream's physical table under strictUpstreams. Success is only
-         // possible if the reference resolved (strict forbids the live fallback).
-         const rollupRun = await buildOneAlone({
-            buildInstructions: {
-               sources: [
-                  {
-                     sourceEntityId: rollup.sourceEntityId,
-                     sourceID: rollup.sourceID,
-                     materializedTableId: "mt-rollup",
-                     physicalTableName: "orders_rollup_built",
-                     realization: "COPY",
-                  },
-               ],
-               referenceManifest: [
-                  {
-                     sourceEntityId: base.sourceEntityId,
-                     physicalTableName: UPSTREAM_TABLE,
-                  },
-               ],
-               strictUpstreams: true,
-            },
-         });
-         expect(rollupRun.status).toBe("MANIFEST_FILE_READY");
-
-         // Cleanup both physical tables and records.
-         await deleteRun(rollupRun.id as string, true);
-         // Drop the upstream table via a fresh no-op build+drop is overkill;
-         // the environment teardown removes the project. Best-effort direct drop
-         // is unnecessary for correctness here.
-      },
-      { timeout: 120_000 },
-   );
-
-   it(
-      "fails loudly when a strict upstream is neither built nor referenced",
-      async () => {
-         const sources = await planSourcesByName();
-         const rollup = sources["orders_rollup"];
-         expect(rollup).toBeDefined();
-
-         // Build `orders_rollup` ALONE under strictUpstreams with NO
-         // referenceManifest for its upstream -> the compile throws
-         // runtime-manifest-strict-miss and the run FAILS.
-         const run = await buildOneAlone({
-            buildInstructions: {
-               sources: [
-                  {
-                     sourceEntityId: rollup.sourceEntityId,
-                     sourceID: rollup.sourceID,
-                     materializedTableId: "mt-rollup-strict",
-                     physicalTableName: "orders_rollup_strict",
-                     realization: "COPY",
-                  },
-               ],
-               strictUpstreams: true,
-            },
-         });
-
-         expect(run.status).toBe("FAILED");
-         expect(String(run.error ?? "")).toContain("manifest");
-
-         await deleteRun(run.id as string, true);
-      },
-      { timeout: 120_000 },
-   );
-});

package/tests/integration/materialization/schedule_edit.integration.spec.ts

@@ -1,126 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const PROJECT_NAME = "sched-edit-project";
-const PACKAGE_NAME = "persist-schedule-test";
-const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
-
-// Exercises the editable-schedule PATCH path: materialization.schedule and
-// scope are writable via updatePackage, validated with the same rules a publish
-// enforces (schedule requires scope: version, valid 5-field cron), and
-// persisted to publisher.json.
-describe("Editable materialization schedule (PATCH)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-
-      const fixtureDir = path.resolve(
-         __dirname,
-         "../../fixtures/persist-schedule-test",
-      );
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PROJECT_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         throw new Error(
-            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
-         );
-      }
-
-      const deadline = Date.now() + 30_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}`);
-         if (res.ok) break;
-         await new Promise((r) => setTimeout(r, 500));
-      }
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   async function getPackage(): Promise<Record<string, unknown>> {
-      const res = await fetch(`${baseUrl}${API}`);
-      expect(res.status).toBe(200);
-      return (await res.json()) as Record<string, unknown>;
-   }
-
-   function scheduleOf(
-      pkg: Record<string, unknown>,
-   ): string | null | undefined {
-      return (pkg.materialization as { schedule?: string | null } | null)
-         ?.schedule;
-   }
-
-   async function patch(body: Record<string, unknown>): Promise<Response> {
-      return fetch(`${baseUrl}${API}`, {
-         method: "PATCH",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({ name: PACKAGE_NAME, ...body }),
-      });
-   }
-
-   it("starts from the fixture's version-scoped schedule", async () => {
-      const pkg = await getPackage();
-      expect(pkg.scope).toBe("version");
-      expect(scheduleOf(pkg)).toBe("0 6 * * *");
-   });
-
-   it("edits and persists a new schedule", async () => {
-      const res = await patch({
-         materialization: { schedule: "*/30 * * * *" },
-      });
-      expect(res.ok).toBe(true);
-      expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
-   });
-
-   it("rejects an invalid cron (400) and leaves the schedule unchanged", async () => {
-      const res = await patch({ materialization: { schedule: "not a cron" } });
-      expect(res.status).toBe(400);
-      expect(scheduleOf(await getPackage())).toBe("*/30 * * * *");
-   });
-
-   it("rejects a schedule without version scope (400)", async () => {
-      const res = await patch({
-         scope: "package",
-         materialization: { schedule: "0 6 * * *" },
-      });
-      expect(res.status).toBe(400);
-      // Prior good state preserved (scope + schedule both unchanged).
-      const pkg = await getPackage();
-      expect(pkg.scope).toBe("version");
-      expect(scheduleOf(pkg)).toBe("*/30 * * * *");
-   });
-
-   it("clears the schedule with schedule: null", async () => {
-      const res = await patch({ materialization: { schedule: null } });
-      expect(res.ok).toBe(true);
-      expect(scheduleOf(await getPackage())).toBeNull();
-   });
-});

package/tests/integration/materialization/scheduler.integration.spec.ts

@@ -1,200 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import fsp from "fs/promises";
-import os from "os";
-import path from "path";
-import { fileURLToPath } from "url";
-import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
-import { MaterializationService } from "../../../src/service/materialization_service";
-import { environmentStore } from "../../../src/server";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const PROJECT_NAME = "sched-project";
-const PACKAGE_NAME = "persist-schedule-test";
-const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
-
-const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
-
-// Deterministic clock for the scheduler: we call tick(nowMs) explicitly, so the
-// fixture's daily cron ("0 6 * * *") fires without waiting on a real minute.
-const T0 = Date.parse("2026-01-01T00:00:00Z");
-const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
-
-describe("MaterializationScheduler (E2E, real build)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-   let scheduler: MaterializationScheduler;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-
-      const fixtureDir = path.resolve(
-         __dirname,
-         "../../fixtures/persist-schedule-test",
-      );
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PROJECT_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         throw new Error(
-            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
-         );
-      }
-
-      // Wait for the package to load (so it's in the environmentStore the
-      // scheduler sweeps).
-      const deadline = Date.now() + 30_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}`);
-         if (res.ok) break;
-         await new Promise((r) => setTimeout(r, 500));
-      }
-
-      // Drive the real scheduler against the server's real environmentStore and
-      // a real MaterializationService (shares the same repository). We call
-      // tick() by hand with a controlled clock rather than relying on the timer.
-      scheduler = new MaterializationScheduler(
-         environmentStore,
-         new MaterializationService(environmentStore),
-         { tickIntervalMs: 60_000, maxFiresPerTick: 10 },
-      );
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   async function listMaterializations(): Promise<
-      Array<Record<string, unknown>>
-   > {
-      const res = await fetch(`${baseUrl}${API}/materializations`);
-      expect(res.status).toBe(200);
-      return (await res.json()) as Array<Record<string, unknown>>;
-   }
-
-   function triggerOf(m: Record<string, unknown>): string | undefined {
-      return (m.metadata as { trigger?: string } | null)?.trigger;
-   }
-
-   async function pollTerminal(id: string): Promise<Record<string, unknown>> {
-      const deadline = Date.now() + 60_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}/materializations/${id}`);
-         expect(res.status).toBe(200);
-         const m = (await res.json()) as Record<string, unknown>;
-         if (TERMINAL.includes(m.status as string)) return m;
-         await new Promise((r) => setTimeout(r, 250));
-      }
-      throw new Error(`materialization ${id} did not reach a terminal state`);
-   }
-
-   it(
-      "fires a due scheduled package as a SCHEDULER build that completes",
-      async () => {
-         // Arm on the first tick (nextFire is strictly future, so no fire yet).
-         await scheduler.tick(T0);
-         expect(
-            (await listMaterializations()).filter(
-               (m) => triggerOf(m) === "SCHEDULER",
-            ).length,
-         ).toBe(0);
-
-         // Advance well past the daily cron: the sweep fires one SCHEDULER run.
-         await scheduler.tick(T0 + THREE_DAYS_MS);
-
-         const scheduled = (await listMaterializations()).filter(
-            (m) => triggerOf(m) === "SCHEDULER",
-         );
-         expect(scheduled.length).toBe(1);
-
-         // The real build settles at MANIFEST_FILE_READY, producing the table.
-         const built = await pollTerminal(scheduled[0].id as string);
-         expect(built.status).toBe("MANIFEST_FILE_READY");
-         const entries = (
-            built.manifest as { entries: Record<string, unknown> }
-         ).entries;
-         expect(Object.keys(entries).length).toBe(1);
-         // The SCHEDULER trigger must survive the build commit (not just exist
-         // while the run is in-flight) — otherwise a completed scheduled run is
-         // indistinguishable from a manual one.
-         expect(triggerOf(built)).toBe("SCHEDULER");
-
-         // Cleanup: drop the table + record so the negative case starts clean.
-         const del = await fetch(
-            `${baseUrl}${API}/materializations/${scheduled[0].id}?dropTables=true`,
-            { method: "DELETE" },
-         );
-         expect(del.status).toBe(204);
-      },
-      { timeout: 120_000 },
-   );
-
-   it(
-      "never fires a control-plane-driven package (manifestLocation set)",
-      async () => {
-         // Make the package orchestrated: point it at a (bind-clean) manifest.
-         const manifestFile = path.join(
-            os.tmpdir(),
-            `sched-empty-manifest-${T0}.json`,
-         );
-         await fsp.writeFile(
-            manifestFile,
-            JSON.stringify({
-               builtAt: new Date(T0).toISOString(),
-               strict: false,
-               entries: {},
-            }),
-            "utf8",
-         );
-         const patch = await fetch(`${baseUrl}${API}`, {
-            method: "PATCH",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: PACKAGE_NAME,
-               manifestLocation: manifestFile,
-            }),
-         });
-         expect(patch.ok).toBe(true);
-
-         const before = (await listMaterializations()).length;
-
-         // Even armed + well past the cron, an orchestrated package is skipped
-         // (Guard 2) — the control plane owns its refresh.
-         await scheduler.tick(T0);
-         await scheduler.tick(T0 + THREE_DAYS_MS);
-
-         expect((await listMaterializations()).length).toBe(before);
-
-         await fetch(`${baseUrl}${API}`, {
-            method: "PATCH",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: PACKAGE_NAME,
-               manifestLocation: null,
-            }),
-         });
-      },
-      { timeout: 60_000 },
-   );
-});