@malloy-publisher/server 0.0.235 → 0.0.236

package/tests/integration/materialization/scheduler_recovery.integration.spec.ts

@@ -1,172 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import { CronEvaluator } from "../../../src/service/cron_evaluator";
-import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
-import { MaterializationService } from "../../../src/service/materialization_service";
-import { environmentStore } from "../../../src/server";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const PROJECT_NAME = "sched-recovery-project";
-const PACKAGE_NAME = "persist-schedule-test";
-const CRON = "0 6 * * *"; // the fixture's daily schedule
-const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
-const CONFIG = { tickIntervalMs: 60_000, maxFiresPerTick: 10 };
-const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
-
-// The restart-recovery anchor is only unit-tested against a stubbed
-// getLatestScheduledFireAt. This drives the real sequence end to end: a real
-// SCHEDULER fire lands a row in the store, then a *second* scheduler instance
-// (a simulated process restart — empty in-memory arming state) re-anchors from
-// that row via the real repo query. We assert exactly one catch-up when an
-// occurrence came due during "downtime", and no double-fire when it didn't.
-describe("MaterializationScheduler restart recovery (E2E, real store)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-   let service: MaterializationService;
-   // A fixed anchor for the *first* scheduler's arming clock; the real fire's
-   // created_at (wall clock) is what recovery actually re-anchors from, and we
-   // read it back from the store rather than assuming it.
-   const T0 = Date.parse("2026-01-01T00:00:00Z");
-   const THREE_DAYS_MS = 3 * 24 * 60 * 60 * 1000;
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-      service = new MaterializationService(environmentStore);
-
-      const fixtureDir = path.resolve(
-         __dirname,
-         "../../fixtures/persist-schedule-test",
-      );
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PROJECT_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         throw new Error(
-            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
-         );
-      }
-
-      // Wait for the package to load into the store the scheduler sweeps.
-      const deadline = Date.now() + 30_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}`);
-         if (res.ok) break;
-         await new Promise((r) => setTimeout(r, 500));
-      }
-   });
-
-   afterAll(async () => {
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   function newScheduler(): MaterializationScheduler {
-      // A fresh instance == a process restart: no in-memory arming state, so the
-      // first arm re-anchors from the store.
-      return new MaterializationScheduler(environmentStore, service, CONFIG);
-   }
-
-   async function scheduledCount(): Promise<number> {
-      const res = await fetch(`${baseUrl}${API}/materializations`);
-      expect(res.status).toBe(200);
-      const rows = (await res.json()) as Array<Record<string, unknown>>;
-      return rows.filter(
-         (m) =>
-            (m.metadata as { trigger?: string } | null)?.trigger ===
-            "SCHEDULER",
-      ).length;
-   }
-
-   async function pollAllTerminal(): Promise<void> {
-      const deadline = Date.now() + 90_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}/materializations`);
-         const rows = (await res.json()) as Array<Record<string, unknown>>;
-         if (rows.every((m) => TERMINAL.includes(m.status as string))) return;
-         await new Promise((r) => setTimeout(r, 250));
-      }
-      throw new Error("materializations did not all reach a terminal state");
-   }
-
-   it(
-      "recovers exactly one missed occurrence across a restart, then jumps forward",
-      async () => {
-         const cron = new CronEvaluator();
-
-         // ── Scheduler #1: produce a real baseline SCHEDULER fire. ──
-         const s1 = newScheduler();
-         await s1.tick(T0); // arm (nextFire strictly future -> no fire)
-         expect(await scheduledCount()).toBe(0);
-         await s1.tick(T0 + THREE_DAYS_MS); // an occurrence is due -> one fire
-         expect(await scheduledCount()).toBe(1);
-         await pollAllTerminal();
-
-         // The instant recovery re-anchors from is the newest SCHEDULER row's
-         // created_at (wall clock), read through the same path the scheduler uses.
-         const lastFire = await service.getLatestScheduledFireAt(
-            PROJECT_NAME,
-            PACKAGE_NAME,
-         );
-         expect(lastFire).not.toBeNull();
-         const missedOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
-
-         // ── Scheduler #2: simulated restart with an occurrence missed during
-         // downtime (now is just past the first occurrence after the last fire). ──
-         const s2 = newScheduler();
-         await s2.tick(missedOccurrence + 60_000); // due on first arm -> catch up
-         expect(await scheduledCount()).toBe(2); // exactly one catch-up
-         await pollAllTerminal();
-         // Advanced forward: the same instance does not fire again next tick.
-         await s2.tick(missedOccurrence + 120_000);
-         expect(await scheduledCount()).toBe(2);
-      },
-      { timeout: 200_000 },
-   );
-
-   it(
-      "does not double-fire on a restart when no occurrence was missed",
-      async () => {
-         const cron = new CronEvaluator();
-         const before = await scheduledCount();
-
-         // Latest SCHEDULER fire is the catch-up above (created_at ~ now). The
-         // next occurrence after it is the next 06:00 UTC, hours away.
-         const lastFire = await service.getLatestScheduledFireAt(
-            PROJECT_NAME,
-            PACKAGE_NAME,
-         );
-         expect(lastFire).not.toBeNull();
-         const nextOccurrence = cron.nextAfter(CRON, lastFire!).getTime();
-
-         // ── Scheduler #3: simulated restart shortly after the last fire, before
-         // the next occurrence is due -> arms into the future, no catch-up. ──
-         const s3 = newScheduler();
-         await s3.tick(lastFire!.getTime() + 1_000);
-         expect(await scheduledCount()).toBe(before);
-         expect(lastFire!.getTime() + 1_000).toBeLessThan(nextOccurrence);
-      },
-      { timeout: 120_000 },
-   );
-});

package/tests/integration/materialization/scheduler_transitions.integration.spec.ts

@@ -1,256 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, afterEach, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
-import { MaterializationService } from "../../../src/service/materialization_service";
-import { environmentStore } from "../../../src/server";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const FIXTURES = path.resolve(__dirname, "../../fixtures");
-const SCHEDULED_FIXTURE = path.join(FIXTURES, "persist-schedule-test"); // cron "0 6 * * *", scope: version
-const PERSIST_FIXTURE = path.join(FIXTURES, "persist-test"); // same persist source, no schedule
-
-// Deterministic clock: tick(nowMs) is called explicitly so the daily
-// "0 6 * * *" cron fires without waiting on a wall-clock minute.
-const T0 = Date.parse("2026-01-01T00:00:00.000Z");
-const MIN = 60_000;
-const H = 60 * MIN;
-const D = 24 * H;
-
-const TERMINAL = ["MANIFEST_FILE_READY", "FAILED", "CANCELLED"];
-
-// Integration coverage for the scheduler's state-machine transitions against the
-// real store (#895): the unit spec covers the per-tick guards; these drive the
-// transitions end to end through the REST-created packages the scheduler sweeps.
-// Each test uses its own environment and tears it down in afterEach — the
-// scheduler sweeps every loaded environment, so a leftover env would leak fires
-// into another test (and steal the per-tick cap in the carryover case).
-describe("MaterializationScheduler transitions (integration, real store)", () => {
-   let e2e: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-   let currentEnv: string | null = null;
-
-   beforeAll(async () => {
-      e2e = await startRestE2E();
-      baseUrl = e2e.baseUrl;
-   });
-
-   afterEach(async () => {
-      if (currentEnv) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${currentEnv}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort teardown
-         }
-         currentEnv = null;
-      }
-   });
-
-   afterAll(async () => {
-      await e2e?.stop();
-      e2e = null;
-   });
-
-   // ── helpers ──────────────────────────────────────────────────────────
-
-   function newScheduler(maxFiresPerTick = 10): MaterializationScheduler {
-      return new MaterializationScheduler(
-         environmentStore,
-         new MaterializationService(environmentStore),
-         { tickIntervalMs: 60_000, maxFiresPerTick },
-      );
-   }
-
-   async function createEnv(
-      name: string,
-      packages: Array<{ name: string; location: string }>,
-   ): Promise<void> {
-      const res = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({ name, packages, connections: [] }),
-      });
-      if (!res.ok) {
-         throw new Error(
-            `create env ${name} failed (${res.status}): ${await res.text()}`,
-         );
-      }
-      currentEnv = name;
-   }
-
-   async function waitForPackage(env: string, pkg: string): Promise<void> {
-      const deadline = Date.now() + 30_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(
-            `${baseUrl}/api/v0/environments/${env}/packages/${pkg}`,
-         );
-         if (res.ok) return;
-         await new Promise((r) => setTimeout(r, 500));
-      }
-      throw new Error(`package ${env}/${pkg} did not load in time`);
-   }
-
-   // Set scope: version + a cron schedule on a package (makes persist-test
-   // standalone-schedulable). editingPolicy is true, so the publish-gate rules
-   // run and must pass (version scope + valid cron, no freshness).
-   async function setSchedule(
-      env: string,
-      pkg: string,
-      schedule: string,
-   ): Promise<void> {
-      const res = await fetch(
-         `${baseUrl}/api/v0/environments/${env}/packages/${pkg}`,
-         {
-            method: "PATCH",
-            headers: { "Content-Type": "application/json" },
-            body: JSON.stringify({
-               name: pkg,
-               scope: "version",
-               materialization: { schedule },
-            }),
-         },
-      );
-      if (!res.ok) {
-         throw new Error(
-            `set schedule on ${pkg} failed (${res.status}): ${await res.text()}`,
-         );
-      }
-   }
-
-   type Row = Record<string, unknown>;
-
-   async function envMaterializations(env: string): Promise<Row[]> {
-      const res = await fetch(
-         `${baseUrl}/api/v0/environments/${env}/packages/materializations`,
-      );
-      expect(res.status).toBe(200);
-      return (await res.json()) as Row[];
-   }
-
-   function triggerOf(m: Row): string | undefined {
-      return (m.metadata as { trigger?: string } | null)?.trigger;
-   }
-
-   async function scheduledFireCount(env: string): Promise<number> {
-      return (await envMaterializations(env)).filter(
-         (m) => triggerOf(m) === "SCHEDULER",
-      ).length;
-   }
-
-   // Let every fired build settle so env teardown doesn't race an in-flight one.
-   async function waitAllTerminal(env: string): Promise<void> {
-      const deadline = Date.now() + 90_000;
-      while (Date.now() < deadline) {
-         const rows = await envMaterializations(env);
-         if (rows.every((m) => TERMINAL.includes(m.status as string))) return;
-         await new Promise((r) => setTimeout(r, 250));
-      }
-      throw new Error(`materializations in ${env} did not all settle`);
-   }
-
-   // ── transitions ─────────────────────────────────────────────────────
-
-   it(
-      "a cron edit between ticks re-arms from now (no stale fire from the old cadence)",
-      async () => {
-         const ENV = "sched-tx-cron-edit";
-         await createEnv(ENV, [
-            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
-         ]);
-         await waitForPackage(ENV, "persist-schedule-test");
-         const sched = newScheduler();
-
-         // Arm on the original "0 6 * * *" cadence: nextFire = Jan-01 06:00.
-         await sched.tick(T0);
-
-         // Edit the cron to midnight before the old 06:00 occurrence fires.
-         await setSchedule(ENV, "persist-schedule-test", "0 0 * * *");
-
-         // Tick just past the OLD 06:00 occurrence. If the edit didn't re-arm,
-         // the stale nextFire would fire here; instead arm() re-anchors from now
-         // on the new cron (next = Jan-02 00:00), so nothing fires.
-         await sched.tick(T0 + 6 * H + MIN);
-         expect(await scheduledFireCount(ENV)).toBe(0);
-
-         // Past the NEW cron's next occurrence (Jan-02 00:00): it fires, proving
-         // the edit re-armed rather than silently dropping the schedule.
-         await sched.tick(T0 + D + MIN);
-         expect(await scheduledFireCount(ENV)).toBe(1);
-
-         await waitAllTerminal(ENV);
-      },
-      { timeout: 120_000 },
-   );
-
-   // The unload/reload prune + re-anchor transition is covered as a deterministic
-   // unit test (materialization_scheduler.spec.ts): it is pure in-memory arming
-   // state, and driving it here via a package delete + same-name re-add races the
-   // package-install staging rename on Windows (POSIX is fine), which is unrelated
-   // to the scheduler behavior under test.
-
-   it(
-      "a fired occurrence advances to the next occurrence (fires once per occurrence, not every tick)",
-      async () => {
-         const ENV = "sched-tx-cadence";
-         await createEnv(ENV, [
-            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
-         ]);
-         await waitForPackage(ENV, "persist-schedule-test");
-         const sched = newScheduler();
-
-         await sched.tick(T0); // arm
-         await sched.tick(T0 + 6 * H + MIN); // due -> fire once
-         expect(await scheduledFireCount(ENV)).toBe(1);
-         await waitAllTerminal(ENV);
-
-         // Immediately after, still inside the same daily window: the scheduler
-         // advanced nextFire to the next occurrence, so it does NOT re-fire every
-         // tick. (The advance is outcome-independent — a FAILED fire advances the
-         // same way; that isolation is covered in the scheduler unit spec.)
-         await sched.tick(T0 + 6 * H + 2 * MIN);
-         expect(await scheduledFireCount(ENV)).toBe(1);
-
-         // The next day's 06:00 occurrence fires again.
-         await sched.tick(T0 + D + 6 * H + MIN);
-         expect(await scheduledFireCount(ENV)).toBe(2);
-         await waitAllTerminal(ENV);
-      },
-      { timeout: 180_000 },
-   );
-
-   it(
-      "maxFiresPerTick caps a tick and the capped package fires on the next tick",
-      async () => {
-         const ENV = "sched-tx-cap";
-         await createEnv(ENV, [
-            { name: "persist-schedule-test", location: SCHEDULED_FIXTURE },
-            { name: "persist-test", location: PERSIST_FIXTURE },
-         ]);
-         await waitForPackage(ENV, "persist-schedule-test");
-         await waitForPackage(ENV, "persist-test");
-         // Make the second package standalone-schedulable on the same cadence.
-         await setSchedule(ENV, "persist-test", "0 6 * * *");
-
-         const sched = newScheduler(1); // cap = 1 fire per tick
-
-         await sched.tick(T0); // arm both
-         // Both due at 06:00, but the cap lets only one fire this tick.
-         await sched.tick(T0 + 6 * H + MIN);
-         expect(await scheduledFireCount(ENV)).toBe(1);
-
-         // The capped package is still due and fires on the following tick.
-         await sched.tick(T0 + 6 * H + 2 * MIN);
-         expect(await scheduledFireCount(ENV)).toBe(2);
-
-         await waitAllTerminal(ENV);
-      },
-      { timeout: 180_000 },
-   );
-});

package/tests/integration/materialization/scheduler_wiring.integration.spec.ts

@@ -1,143 +0,0 @@
-/// <reference types="bun-types" />
-
-import { afterAll, beforeAll, describe, expect, it } from "bun:test";
-import path from "path";
-import { fileURLToPath } from "url";
-import type { MaterializationScheduler } from "../../../src/service/materialization_scheduler";
-import { MaterializationService } from "../../../src/service/materialization_service";
-import {
-   environmentStore,
-   startMaterializationSchedulerFromEnv,
-} from "../../../src/server";
-import { RestE2EEnv, startRestE2E } from "../../harness/rest_e2e";
-
-const __filename = fileURLToPath(import.meta.url);
-const __dirname = path.dirname(__filename);
-
-const PROJECT_NAME = "sched-wiring-project";
-const PACKAGE_NAME = "persist-schedule-test";
-const API = `/api/v0/environments/${PROJECT_NAME}/packages/${PACKAGE_NAME}`;
-
-const ENABLE = "PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER";
-const INTERVAL = "PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS";
-
-// The other scheduler tests hand-construct MaterializationScheduler and call
-// tick() by hand, bypassing the env-var → getMaterializationSchedulerConfig →
-// construct → start()/unref() → real-timer path an operator actually runs. This
-// exercises that wiring: set the flag + a short interval, start the scheduler
-// through the same seam server.ts uses at boot, and observe a build fire through
-// the real timer (not a hand-driven tick).
-describe("MaterializationScheduler wiring (E2E, real env-var timer)", () => {
-   let env: (RestE2EEnv & { stop(): Promise<void> }) | null = null;
-   let baseUrl: string;
-   let scheduler: MaterializationScheduler | null = null;
-   const prev: Record<string, string | undefined> = {};
-
-   beforeAll(async () => {
-      env = await startRestE2E();
-      baseUrl = env.baseUrl;
-
-      const fixtureDir = path.resolve(
-         __dirname,
-         "../../fixtures/persist-schedule-test",
-      );
-      const createRes = await fetch(`${baseUrl}/api/v0/environments`, {
-         method: "POST",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PROJECT_NAME,
-            packages: [{ name: PACKAGE_NAME, location: fixtureDir }],
-            connections: [],
-         }),
-      });
-      if (!createRes.ok) {
-         throw new Error(
-            `Failed to create test project (${createRes.status}): ${await createRes.text()}`,
-         );
-      }
-
-      // Wait for the package to load, then switch it to an every-minute cron so
-      // the real timer fires within the test window (the fixture ships a daily
-      // cron, which a wall-clock test can't wait on).
-      const deadline = Date.now() + 30_000;
-      while (Date.now() < deadline) {
-         const res = await fetch(`${baseUrl}${API}`);
-         if (res.ok) break;
-         await new Promise((r) => setTimeout(r, 500));
-      }
-      const patch = await fetch(`${baseUrl}${API}`, {
-         method: "PATCH",
-         headers: { "Content-Type": "application/json" },
-         body: JSON.stringify({
-            name: PACKAGE_NAME,
-            materialization: { schedule: "* * * * *" },
-         }),
-      });
-      expect(patch.ok).toBe(true);
-   });
-
-   afterAll(async () => {
-      scheduler?.stop();
-      // Restore env so the shared test process isn't left with the flag set.
-      for (const key of [ENABLE, INTERVAL]) {
-         if (prev[key] === undefined) delete process.env[key];
-         else process.env[key] = prev[key];
-      }
-      if (baseUrl) {
-         try {
-            await fetch(`${baseUrl}/api/v0/environments/${PROJECT_NAME}`, {
-               method: "DELETE",
-            });
-         } catch {
-            // best-effort
-         }
-      }
-      await env?.stop();
-      env = null;
-   });
-
-   it(
-      "boots the scheduler from env config and fires a build through the real timer",
-      async () => {
-         // Enable via the real env-var path and use the minimum 1s sweep so the
-         // every-minute cron is picked up promptly once its boundary passes.
-         prev[ENABLE] = process.env[ENABLE];
-         prev[INTERVAL] = process.env[INTERVAL];
-         process.env[ENABLE] = "true";
-         process.env[INTERVAL] = "1000";
-
-         scheduler = startMaterializationSchedulerFromEnv(
-            environmentStore,
-            new MaterializationService(environmentStore),
-         );
-         // The config path must have constructed + started a scheduler (a null
-         // here would mean the enable flag wasn't honored).
-         expect(scheduler).not.toBeNull();
-
-         // Wait for the real timer to fire a SCHEDULER build and for it to build.
-         const deadline = Date.now() + 150_000;
-         let built: Record<string, unknown> | null = null;
-         while (Date.now() < deadline && !built) {
-            const res = await fetch(`${baseUrl}${API}/materializations`);
-            const rows = (await res.json()) as Array<Record<string, unknown>>;
-            built =
-               rows.find(
-                  (m) =>
-                     (m.metadata as { trigger?: string } | null)?.trigger ===
-                        "SCHEDULER" && m.status === "MANIFEST_FILE_READY",
-               ) ?? null;
-            if (!built) await new Promise((r) => setTimeout(r, 1_000));
-         }
-
-         expect(built).not.toBeNull();
-         const entries = (
-            built!.manifest as { entries: Record<string, unknown> }
-         ).entries;
-         expect(Object.keys(entries).length).toBe(1);
-
-         // Stop the timer before the test ends so it can't fire into other files.
-         scheduler!.stop();
-      },
-      { timeout: 180_000 },
-   );
-});