@malloy-publisher/server 0.0.235 → 0.0.236

package/src/service/materialization_schedule_surface.spec.ts

@@ -1,256 +0,0 @@
-import { afterEach, beforeEach, describe, expect, it } from "bun:test";
-import * as fs from "fs/promises";
-import * as os from "os";
-import * as path from "path";
-
-import { Environment } from "./environment";
-
-/**
- * The publisher must surface a non-null `materialization` object on a loaded
- * package's metadata so the control plane can treat object-present as the
- * authoritative manifest policy ("this is what the manifest says") and
- * object-absent as "metadata not loaded this request" — never as a schedule
- * removal. A metadata PATCH must not drop that policy from the in-memory
- * metadata. Regression coverage for the re-materialization schedule self-wipe
- * (docs/bugs/materialization-schedule-self-wipe.md).
- *
- * Runs against a real `Environment` + real `Package.create` over temp dirs.
- */
-describe("materialization schedule surfacing", () => {
-   const MODEL = `source: ones is duckdb.sql("SELECT 1 as x")\n`;
-   let rootDir: string;
-   let envPath: string;
-
-   async function writePackageDir(
-      manifest: Record<string, unknown>,
-   ): Promise<void> {
-      const dir = path.join(envPath, "pkg");
-      await fs.mkdir(dir, { recursive: true });
-      await fs.writeFile(
-         path.join(dir, "publisher.json"),
-         JSON.stringify({ name: "pkg", description: "fixture", ...manifest }),
-      );
-      await fs.writeFile(path.join(dir, "model.malloy"), MODEL);
-   }
-
-   beforeEach(async () => {
-      rootDir = await fs.mkdtemp(path.join(os.tmpdir(), "publisher-mat-"));
-      envPath = path.join(rootDir, "env");
-      await fs.mkdir(envPath, { recursive: true });
-   });
-
-   afterEach(async () => {
-      await fs.rm(rootDir, { recursive: true, force: true }).catch(() => {});
-   });
-
-   it(
-      "surfaces the manifest's materialization.schedule on load",
-      async () => {
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({ materialization: { schedule: "0 6 * * *" } });
-         await env.addPackage("pkg");
-
-         const pkg = await env.getPackage("pkg", false);
-         expect(pkg.getPackageMetadata().materialization).toEqual({
-            schedule: "0 6 * * *",
-            freshness: null,
-            queryMetadata: null,
-         });
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "surfaces the manifest's materialization.freshness on load",
-      async () => {
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({
-            materialization: {
-               freshness: { window: "24h", fallback: "stale_ok" },
-            },
-         });
-         await env.addPackage("pkg");
-
-         // Freshness rides the same channel as schedule: present-and-verbatim
-         // when declared, null when unset — the control plane owns the
-         // scheduling/gating logic and only needs the values surfaced.
-         const pkg = await env.getPackage("pkg", false);
-         expect(pkg.getPackageMetadata().materialization).toEqual({
-            schedule: null,
-            freshness: { window: "24h", fallback: "stale_ok" },
-            queryMetadata: null,
-         });
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "surfaces a non-null materialization object even when the manifest declares none",
-      async () => {
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({});
-         await env.addPackage("pkg");
-
-         // Object present with a null schedule — NOT a null object — so the
-         // control plane reads it as an authoritative "no schedule declared",
-         // not as "metadata unavailable".
-         const pkg = await env.getPackage("pkg", false);
-         expect(pkg.getPackageMetadata().materialization).toEqual({
-            schedule: null,
-            freshness: null,
-         });
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "preserves the materialization policy across a metadata PATCH",
-      async () => {
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({ materialization: { schedule: "0 6 * * *" } });
-         await env.addPackage("pkg");
-
-         // A description-only PATCH must not wipe the schedule from the
-         // in-memory metadata. The bug: setPackageMetadata replaces the whole
-         // object, so a later getPackage reported no schedule and the control
-         // plane misread the gap as a removal and cleared the cadence.
-         const updated = await env.updatePackage("pkg", {
-            name: "pkg",
-            description: "updated",
-         });
-         expect(updated.materialization).toEqual({
-            schedule: "0 6 * * *",
-            freshness: null,
-            queryMetadata: null,
-         });
-
-         const pkg = await env.getPackage("pkg", false);
-         expect(pkg.getPackageMetadata().materialization).toEqual({
-            schedule: "0 6 * * *",
-            freshness: null,
-            queryMetadata: null,
-         });
-      },
-      { timeout: 20000 },
-   );
-
-   async function readManifest(): Promise<Record<string, unknown>> {
-      return JSON.parse(
-         await fs.readFile(path.join(envPath, "pkg", "publisher.json"), "utf8"),
-      );
-   }
-
-   it(
-      "writes scope to both homes so the manifest it authors still loads",
-      async () => {
-         // A scope PATCH used to write only the manifest root. With scope also
-         // living in the materialization block, that authored a manifest whose
-         // two homes disagreed — which the loader refuses, so the package
-         // survived the PATCH and then failed on the next restart.
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({
-            materialization: {
-               scope: "package",
-               queryMetadata: { team: "fin" },
-            },
-         });
-         await env.addPackage("pkg");
-
-         await env.updatePackage("pkg", { name: "pkg", scope: "version" });
-
-         const manifest = await readManifest();
-         expect(manifest.scope).toBe("version");
-         expect(manifest.materialization).toMatchObject({
-            scope: "version",
-            queryMetadata: { team: "fin" },
-         });
-
-         // The real assertion: it loads again.
-         const reloaded = await Environment.create("testEnv", envPath, []);
-         await reloaded.addPackage("pkg");
-         const pkg = await reloaded.getPackage("pkg", false);
-         expect(pkg.getPackageMetadata().scope).toBe("version");
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "keeps the envelope scope a materialization PATCH cannot express",
-      async () => {
-         // The wire materialization block has no `scope`, so a PATCH that sends
-         // one must not be read as "the author dropped it".
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({ materialization: { scope: "version" } });
-         await env.addPackage("pkg");
-
-         await env.updatePackage("pkg", {
-            name: "pkg",
-            materialization: { schedule: "0 6 * * *" },
-         });
-
-         const manifest = await readManifest();
-         expect(manifest.materialization).toMatchObject({
-            scope: "version",
-            schedule: "0 6 * * *",
-         });
-         expect(manifest.scope).toBe("version");
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "keeps queryMetadata a schedule PATCH said nothing about",
-      async () => {
-         // The block is replaced wholesale, which is right for the policy the
-         // caller is setting — but queryMetadata is orthogonal to it, so a
-         // client that sets a schedule without re-sending the tags would
-         // silently untag every statement the package's builds issue. The UI
-         // and the control plane are separate clients; preserving it here fixes
-         // all of them at once.
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({
-            materialization: {
-               scope: "version",
-               queryMetadata: { team: "finance" },
-            },
-         });
-         await env.addPackage("pkg");
-
-         await env.updatePackage("pkg", {
-            name: "pkg",
-            materialization: { schedule: "0 6 * * *" },
-         });
-
-         expect((await readManifest()).materialization).toMatchObject({
-            schedule: "0 6 * * *",
-            queryMetadata: { team: "finance" },
-         });
-      },
-      { timeout: 20000 },
-   );
-
-   it(
-      "still lets an explicit null clear queryMetadata",
-      async () => {
-         // Preserved-on-omission must not make it unclearable.
-         const env = await Environment.create("testEnv", envPath, []);
-         await writePackageDir({
-            materialization: {
-               scope: "version",
-               queryMetadata: { team: "finance" },
-            },
-         });
-         await env.addPackage("pkg");
-
-         await env.updatePackage("pkg", {
-            name: "pkg",
-            materialization: { schedule: "0 6 * * *", queryMetadata: null },
-         });
-
-         expect(
-            (await readManifest()).materialization as Record<string, unknown>,
-         ).toMatchObject({ schedule: "0 6 * * *", queryMetadata: null });
-      },
-      { timeout: 20000 },
-   );
-});

package/src/service/materialization_scheduler.spec.ts

@@ -1,289 +0,0 @@
-import { describe, expect, it } from "bun:test";
-import { MaterializationConflictError } from "../errors";
-import type { CronEvaluator } from "./cron_evaluator";
-import type { EnvironmentStore } from "./environment_store";
-import type { MaterializationService } from "./materialization_service";
-import { MaterializationScheduler } from "./materialization_scheduler";
-
-// ── Fakes ────────────────────────────────────────────────────────────────
-
-interface FakePkgOpts {
-   schedule?: string | null;
-   manifestLocation?: string | null;
-   policyWarnings?: string[];
-}
-
-function fakePackage(name: string, opts: FakePkgOpts = {}) {
-   const {
-      schedule = null,
-      manifestLocation = null,
-      policyWarnings = [],
-   } = opts;
-   return {
-      getPackageName: () => name,
-      getPackageMetadata: () => ({
-         materialization: schedule ? { schedule } : null,
-         manifestLocation,
-      }),
-      persistencePolicyWarnings: () => policyWarnings,
-   };
-}
-
-function fakeEnv(name: string, pkgs: ReturnType<typeof fakePackage>[]) {
-   return {
-      getEnvironmentName: () => name,
-      getLoadedPackages: () => pkgs,
-   };
-}
-
-function fakeStore(envs: ReturnType<typeof fakeEnv>[]): EnvironmentStore {
-   return {
-      getLoadedEnvironments: () => envs,
-   } as unknown as EnvironmentStore;
-}
-
-interface FireCall {
-   env: string;
-   pkg: string;
-   opts: { forceRefresh?: boolean; trigger?: string };
-}
-
-function fakeService(
-   opts: {
-      throwConflict?: boolean;
-      throwError?: boolean;
-      // Newest recorded SCHEDULER fire, used by the restart-recovery anchor.
-      lastFireAt?: Date | null;
-   } = {},
-) {
-   const calls: FireCall[] = [];
-   const service = {
-      calls,
-      createMaterialization: (
-         env: string,
-         pkg: string,
-         o: { forceRefresh?: boolean; trigger?: string },
-      ) => {
-         calls.push({ env, pkg, opts: o });
-         if (opts.throwConflict) {
-            return Promise.reject(new MaterializationConflictError("busy"));
-         }
-         if (opts.throwError) {
-            return Promise.reject(new Error("boom"));
-         }
-         return Promise.resolve({});
-      },
-      getLatestScheduledFireAt: () => Promise.resolve(opts.lastFireAt ?? null),
-   };
-   return service as unknown as MaterializationService & { calls: FireCall[] };
-}
-
-// A deterministic cron: nextAfter is always `from + 60s`; isValid configurable.
-function fakeCron(
-   isValid: (expr: string) => boolean = () => true,
-): CronEvaluator {
-   return {
-      isValid,
-      nextAfter: (_expr: string, from: Date) =>
-         new Date(from.getTime() + 60_000),
-   } as unknown as CronEvaluator;
-}
-
-const CONFIG = { tickIntervalMs: 60_000, maxFiresPerTick: 10 };
-
-function makeScheduler(
-   pkgs: ReturnType<typeof fakePackage>[],
-   service: ReturnType<typeof fakeService>,
-   cron: CronEvaluator = fakeCron(),
-   config = CONFIG,
-) {
-   const store = fakeStore([fakeEnv("env1", pkgs)]);
-   return new MaterializationScheduler(store, service, config, cron);
-}
-
-// ── Tests ────────────────────────────────────────────────────────────────
-
-describe("MaterializationScheduler", () => {
-   const t0 = 1_000_000_000_000; // fixed epoch ms
-   const dueLater = t0 + 60_001; // just past the armed nextFire (t0 + 60s)
-
-   it("does not fire on the arming tick (nextFire is strictly future)", async () => {
-      const service = fakeService();
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("recovers a missed occurrence on first arm, firing once then jumping forward", async () => {
-      // A prior SCHEDULER fire well in the past: anchoring nextFire to
-      // nextAfter(lastFire) lands before t0, so the occurrence missed during
-      // "downtime" is due on the very first tick (a restart recovering).
-      const service = fakeService({ lastFireAt: new Date(t0 - 10 * 60_000) });
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0);
-      expect(service.calls.length).toBe(1); // one catch-up fire
-      // Jumped forward to nextAfter(now); not due again on the next tick.
-      await sched.tick(t0 + 1);
-      expect(service.calls.length).toBe(1);
-   });
-
-   it("does not catch up a never-fired schedule on first arm", async () => {
-      // No prior SCHEDULER fire on record -> anchor is now -> strictly future,
-      // so a freshly-scheduled (never-fired) package does not fire on arm.
-      const service = fakeService({ lastFireAt: null });
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("fires a due, valid, standalone package as SCHEDULER + forceRefresh", async () => {
-      const service = fakeService();
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0); // arm
-      await sched.tick(dueLater); // due -> fire
-      expect(service.calls).toEqual([
-         {
-            env: "env1",
-            pkg: "p",
-            opts: { forceRefresh: true, trigger: "SCHEDULER" },
-         },
-      ]);
-   });
-
-   it("skips a package with no schedule", async () => {
-      const service = fakeService();
-      const sched = makeScheduler([fakePackage("p", {})], service);
-      await sched.tick(t0);
-      await sched.tick(dueLater);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("skips an orchestrated package (manifestLocation set) — Guard 2", async () => {
-      const service = fakeService();
-      const sched = makeScheduler(
-         [
-            fakePackage("p", {
-               schedule: "* * * * *",
-               manifestLocation: "gs://cp/manifest.json",
-            }),
-         ],
-         service,
-      );
-      await sched.tick(t0);
-      await sched.tick(dueLater);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("skips a package whose persistence policy is invalid", async () => {
-      const service = fakeService();
-      const sched = makeScheduler(
-         [
-            fakePackage("p", {
-               schedule: "* * * * *",
-               policyWarnings: ["schedule requires scope: version"],
-            }),
-         ],
-         service,
-      );
-      await sched.tick(t0);
-      await sched.tick(dueLater);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("skips an invalid cron", async () => {
-      const service = fakeService();
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "not a cron" })],
-         service,
-         fakeCron(() => false),
-      );
-      await sched.tick(t0);
-      await sched.tick(dueLater);
-      expect(service.calls.length).toBe(0);
-   });
-
-   it("coalesces when a materialization is already active (no throw)", async () => {
-      const service = fakeService({ throwConflict: true });
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0);
-      await sched.tick(dueLater); // due -> fire attempt -> conflict, swallowed
-      expect(service.calls.length).toBe(1); // attempted once
-      // Advanced past this occurrence: not due again until the next window.
-      await sched.tick(dueLater + 1);
-      expect(service.calls.length).toBe(1);
-   });
-
-   it("isolates an unexpected fire error (sweep keeps going)", async () => {
-      const service = fakeService({ throwError: true });
-      const sched = makeScheduler(
-         [fakePackage("p", { schedule: "* * * * *" })],
-         service,
-      );
-      await sched.tick(t0);
-      await expect(sched.tick(dueLater)).resolves.toBeUndefined();
-      expect(service.calls.length).toBe(1);
-   });
-
-   it("caps fires per tick (stampede guard)", async () => {
-      const service = fakeService();
-      const pkgs = [
-         fakePackage("a", { schedule: "* * * * *" }),
-         fakePackage("b", { schedule: "* * * * *" }),
-         fakePackage("c", { schedule: "* * * * *" }),
-      ];
-      const sched = makeScheduler(pkgs, service, fakeCron(), {
-         tickIntervalMs: 60_000,
-         maxFiresPerTick: 2,
-      });
-      await sched.tick(t0); // arm all
-      await sched.tick(dueLater); // all due, but cap = 2
-      expect(service.calls.length).toBe(2);
-      // The capped one is still due and fires on the next tick.
-      await sched.tick(dueLater + 1);
-      expect(service.calls.length).toBe(3);
-   });
-
-   it("prunes arming state when a package unloads, then re-anchors on reload without a stale fire", async () => {
-      // A never-fired schedule (fakeService default lastFireAt = null), so a
-      // fresh arm anchors from `now` and does not catch up.
-      const service = fakeService();
-      const p = fakePackage("p", { schedule: "* * * * *" });
-      // makeScheduler closes over this array via getLoadedPackages, so mutating
-      // it simulates the package leaving and re-entering the loaded set.
-      const loaded = [p];
-      const sched = makeScheduler(loaded, service);
-
-      await sched.tick(t0); // arm: nextFire = t0 + 60s
-
-      // Unload: the package leaves the loaded set, so the sweep prunes its state
-      // (tick's not-seen cleanup) instead of leaving a stale nextFire behind.
-      loaded.length = 0;
-      await sched.tick(dueLater); // past the old nextFire, but the package is gone
-      expect(service.calls.length).toBe(0);
-
-      // Reload: with the state pruned, arm is fresh — anchored from `now`, so the
-      // pre-unload nextFire can't resurface as a spurious catch-up.
-      loaded.push(p);
-      await sched.tick(dueLater + 1); // now < fresh nextFire -> no fire
-      expect(service.calls.length).toBe(0);
-
-      // It is genuinely re-armed (not inert): the next occurrence still fires.
-      await sched.tick(dueLater + 1 + 60_001);
-      expect(service.calls.length).toBe(1);
-   });
-});