@malloy-publisher/server 0.0.227 → 0.0.229

package/src/service/materialization_scheduler.ts

@@ -0,0 +1,304 @@
+import type { MaterializationSchedulerConfig } from "../config";
+import { MaterializationConflictError } from "../errors";
+import { logger } from "../logger";
+import { recordScheduledFire } from "../materialization_metrics";
+import { CronEvaluator } from "./cron_evaluator";
+import type { EnvironmentStore } from "./environment_store";
+import type { MaterializationService } from "./materialization_service";
+
+/** Per-package arming state, keyed by `${environmentName}::${packageName}`. */
+interface ScheduleState {
+   /** The cron the state was armed from; a change re-arms `nextFireAtMs`. */
+   schedule: string;
+   /** Epoch ms of the next due fire (`cron.nextAfter`, strictly future when armed). */
+   nextFireAtMs: number;
+   /** Epoch ms of the last fire attempt, or null if never fired. */
+   lastFiredAtMs: number | null;
+}
+
+/**
+ * Standalone materialization scheduler: fires a loaded package's
+ * `materialization.schedule` cron so an open-source publisher (no control plane)
+ * can rebuild on a cadence. Modeled on {@link PackageMemoryGovernor} — a
+ * config-driven `setInterval` with `start`/`stop` and one `tick`.
+ *
+ * **Orchestration safety (two independent guards):**
+ *   1. Disabled by default — only constructed when
+ *      {@link getMaterializationSchedulerConfig} returns non-null
+ *      (`PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER`), which an orchestrated
+ *      deployment never sets, so the scheduler never runs there.
+ *   2. Even when enabled, a package with a configured `manifestLocation` (i.e.
+ *      control-plane-driven) is skipped — the CP already fires it, so we never
+ *      double-drive.
+ *
+ * These guards are **independent and both load-bearing — never collapse them to
+ * just the `manifestLocation` check.** A control-plane-loaded package that is
+ * *serving live* has `manifestLocation === null`, so Guard 2 does not cover it;
+ * only Guard 1 (the opt-in flag, which an orchestrated worker must never set —
+ * see {@link getMaterializationSchedulerConfig}) keeps the scheduler off there.
+ *
+ * It only sweeps **already-loaded** packages (never forces a load), fires the
+ * existing `createMaterialization` auto-run path tagged `trigger=SCHEDULER`, and
+ * only fires packages whose persistence policy is valid (`scope: version`, no
+ * `freshness` — the same rule a publish enforces). Arming state is in memory,
+ * but on the first arm after a (re)start it re-anchors from the newest recorded
+ * `SCHEDULER` fire (see {@link arm}), so an occurrence that came due during
+ * downtime fires exactly once and then jumps forward — recovery matches the
+ * control plane's persisted-`next_fire_at` behavior rather than silently
+ * skipping. A never-fired schedule still does not fire on the arming tick
+ * (`nextAfter(now)` is strictly future).
+ */
+export class MaterializationScheduler {
+   private timer: ReturnType<typeof setInterval> | null = null;
+   private readonly cron: CronEvaluator;
+   private readonly state = new Map<string, ScheduleState>();
+
+   constructor(
+      private readonly environmentStore: EnvironmentStore,
+      private readonly materializationService: MaterializationService,
+      private readonly config: MaterializationSchedulerConfig,
+      cron?: CronEvaluator,
+   ) {
+      this.cron = cron ?? new CronEvaluator();
+   }
+
+   /** Begin the periodic sweep. Idempotent; the interval is `unref`'d. */
+   public start(): void {
+      if (this.timer !== null) return;
+      this.timer = setInterval(() => {
+         void this.tick();
+      }, this.config.tickIntervalMs);
+      (
+         this.timer as ReturnType<typeof setInterval> & { unref?: () => void }
+      ).unref?.();
+      logger.info(
+         `MaterializationScheduler started (interval=${this.config.tickIntervalMs}ms, maxFiresPerTick=${this.config.maxFiresPerTick})`,
+      );
+   }
+
+   /** Stop the periodic sweep. Idempotent. */
+   public stop(): void {
+      if (this.timer !== null) {
+         clearInterval(this.timer);
+         this.timer = null;
+      }
+   }
+
+   /**
+    * One sweep: arm newly-seen schedules, fire due ones (capped per tick), and
+    * prune state for packages no longer loaded/scheduled. Exposed so tests can
+    * drive it synchronously. Never throws — per-package errors are isolated.
+    */
+   public async tick(nowMs: number = Date.now()): Promise<void> {
+      const seen = new Set<string>();
+      let fired = 0;
+
+      for (const env of this.environmentStore.getLoadedEnvironments()) {
+         const environmentName = env.getEnvironmentName();
+         for (const pkg of env.getLoadedPackages()) {
+            const packageName = pkg.getPackageName();
+            const key = `${environmentName}::${packageName}`;
+            try {
+               const schedule = this.schedulableCron(pkg);
+               if (schedule === null) {
+                  // Not a standalone-schedulable package (no cron, orchestrated,
+                  // or invalid policy/cron) — drop any stale arming state.
+                  this.state.delete(key);
+                  continue;
+               }
+               seen.add(key);
+
+               const armed = await this.arm(
+                  environmentName,
+                  packageName,
+                  key,
+                  schedule,
+                  nowMs,
+               );
+               if (nowMs < armed.nextFireAtMs) continue; // not due yet
+
+               if (fired >= this.config.maxFiresPerTick) {
+                  // Stampede guard: leave it due; it fires on a later tick.
+                  continue;
+               }
+               await this.fire(environmentName, packageName);
+               fired++;
+               // Advance regardless of outcome so a persistent failure retries on
+               // the next cron occurrence (cron cadence), never every tick.
+               armed.nextFireAtMs = this.cron
+                  .nextAfter(schedule, new Date(nowMs))
+                  .getTime();
+               armed.lastFiredAtMs = nowMs;
+            } catch (err) {
+               logger.warn("MaterializationScheduler: package sweep failed", {
+                  environmentName,
+                  packageName,
+                  error: err instanceof Error ? err.message : String(err),
+               });
+            }
+         }
+      }
+
+      for (const key of [...this.state.keys()]) {
+         if (!seen.has(key)) this.state.delete(key);
+      }
+   }
+
+   /**
+    * The package's schedulable cron, or null when it must be skipped: no
+    * `materialization.schedule`; a `manifestLocation` (control-plane-driven —
+    * Guard 2); an invalid persistence policy (e.g. schedule without
+    * `scope: version`, or schedule + freshness — the publish-gate rules); or an
+    * unparseable cron. Skips are debug-logged (the policy issues are already
+    * warned at package load), not warned each tick.
+    */
+   private schedulableCron(pkg: {
+      getPackageMetadata(): {
+         materialization?: { schedule?: string | null } | null;
+         manifestLocation?: string | null;
+      };
+      persistencePolicyWarnings(): string[];
+      getPackageName(): string;
+   }): string | null {
+      const metadata = pkg.getPackageMetadata();
+      const schedule = metadata.materialization?.schedule ?? null;
+      if (!schedule) return null;
+      if (metadata.manifestLocation) return null; // orchestrated — CP fires it
+      if (pkg.persistencePolicyWarnings().length > 0) {
+         logger.debug(
+            "MaterializationScheduler: skipping package with invalid persistence policy",
+            { packageName: pkg.getPackageName() },
+         );
+         return null;
+      }
+      if (!this.cron.isValid(schedule)) {
+         logger.debug("MaterializationScheduler: skipping invalid cron", {
+            packageName: pkg.getPackageName(),
+            schedule,
+         });
+         return null;
+      }
+      return schedule;
+   }
+
+   /**
+    * Return the package's arming state, (re)arming when it is new or its cron
+    * changed.
+    *
+    * **First arm of a package this process** (no in-memory state — a fresh start
+    * or a newly-loaded package): anchor `nextFireAt` to the newest recorded
+    * `SCHEDULER` fire, `nextAfter(lastScheduledFire)`. If that instant is already
+    * past (an occurrence came due while the publisher was down), it fires once
+    * on this tick and then jumps forward to `nextAfter(now)` — exactly one
+    * catch-up, matching the control plane's persisted-`next_fire_at` recovery
+    * contract, never a burst. With no prior `SCHEDULER` fire on record we fall
+    * back to `nextAfter(now)` (strictly future), so a never-fired schedule does
+    * not fire on the arming tick. (Residual divergence: a schedule set while the
+    * scheduler was disabled has no prior fire, so it is not caught up on first
+    * enable.)
+    *
+    * **Cron changed**: re-arm from `nextAfter(now)` — an edited cadence starts
+    * fresh, with no catch-up for the old cron.
+    */
+   private async arm(
+      environmentName: string,
+      packageName: string,
+      key: string,
+      schedule: string,
+      nowMs: number,
+   ): Promise<ScheduleState> {
+      const existing = this.state.get(key);
+      if (existing && existing.schedule === schedule) {
+         return existing;
+      }
+
+      const anchor =
+         existing === undefined
+            ? await this.recoveryAnchor(environmentName, packageName, nowMs)
+            : new Date(nowMs);
+
+      const armed: ScheduleState = {
+         schedule,
+         nextFireAtMs: this.cron.nextAfter(schedule, anchor).getTime(),
+         lastFiredAtMs: existing?.lastFiredAtMs ?? null,
+      };
+      this.state.set(key, armed);
+      return armed;
+   }
+
+   /**
+    * The instant to compute the first `nextFireAt` from after a (re)start: the
+    * newest recorded `SCHEDULER` fire if any, else `now`. Any lookup failure
+    * degrades to `now` (no catch-up) so arming always succeeds.
+    */
+   private async recoveryAnchor(
+      environmentName: string,
+      packageName: string,
+      nowMs: number,
+   ): Promise<Date> {
+      try {
+         const last =
+            await this.materializationService.getLatestScheduledFireAt(
+               environmentName,
+               packageName,
+            );
+         return last ?? new Date(nowMs);
+      } catch (err) {
+         logger.warn(
+            "MaterializationScheduler: could not read last scheduled fire; arming from now",
+            {
+               environmentName,
+               packageName,
+               error: err instanceof Error ? err.message : String(err),
+            },
+         );
+         return new Date(nowMs);
+      }
+   }
+
+   /**
+    * Fire a whole-package scheduled rebuild via the existing auto-run path. An
+    * already-active materialization coalesces (the in-flight build covers this
+    * occurrence); any other error is logged and metered but never propagated.
+    *
+    * **Standalone fires are tables-only.** This path runs a single-phase build
+    * that materializes the package's persist sources into tables. A hosted
+    * (control-plane-driven) deployment runs a two-phase job — tables, then a
+    * second pass over indexed dimensions — so index behavior is a hosted-only
+    * concern that cannot be exercised by the standalone scheduler. Anything a
+    * schedule fires locally is exactly what the `createMaterialization`
+    * auto-run produces on demand; only the `trigger` label differs.
+    */
+   private async fire(
+      environmentName: string,
+      packageName: string,
+   ): Promise<void> {
+      try {
+         await this.materializationService.createMaterialization(
+            environmentName,
+            packageName,
+            { forceRefresh: true, trigger: "SCHEDULER" },
+         );
+         recordScheduledFire("fired");
+         logger.info("MaterializationScheduler: fired scheduled rebuild", {
+            environmentName,
+            packageName,
+         });
+      } catch (err) {
+         if (err instanceof MaterializationConflictError) {
+            recordScheduledFire("conflict");
+            logger.debug(
+               "MaterializationScheduler: skipped fire; a materialization is already active",
+               { environmentName, packageName },
+            );
+            return;
+         }
+         recordScheduledFire("error");
+         logger.warn("MaterializationScheduler: scheduled fire failed", {
+            environmentName,
+            packageName,
+            error: err instanceof Error ? err.message : String(err),
+         });
+      }
+   }
+}

package/src/service/materialization_service.spec.ts

@@ -166,6 +166,7 @@ describe("MaterializationService", () => {
                forceRefresh: true,
                sourceNames: ["orders"],
                mode: "auto",
+               trigger: "ON_DEMAND",
             },
          ]);
       });
@@ -183,10 +184,27 @@ describe("MaterializationService", () => {
                forceRefresh: false,
                sourceNames: null,
                mode: "auto",
+               trigger: "ON_DEMAND",
             },
          );
       });
 
+      it("records trigger=SCHEDULER when the scheduler fires the run", async () => {
+         ctx.repository.getActiveMaterialization.resolves(null);
+         ctx.repository.createMaterialization.resolves(
+            makeMaterialization({ status: "PENDING" }),
+         );
+
+         await ctx.service.createMaterialization("my-env", "pkg", {
+            forceRefresh: true,
+            trigger: "SCHEDULER",
+         });
+
+         expect(
+            ctx.repository.createMaterialization.firstCall.args[3],
+         ).toMatchObject({ mode: "auto", trigger: "SCHEDULER" });
+      });
+
       it("rejects when an active materialization already exists", async () => {
          ctx.repository.getActiveMaterialization.resolves(
             makeMaterialization({
@@ -788,6 +806,110 @@ describe("executeInstructedBuild", () => {
       expect(rootIdx).toBeGreaterThan(midIdx);
    });
 
+   it("seeds a downstream build with the QUOTED upstream reference (case-folding dialect)", async () => {
+      // A carried/seeded upstream (built in a prior run or unit, e.g. via
+      // referenceManifest) must reach the downstream's SQL generation as the
+      // SAME quoted path the builder CREATEd it with; otherwise the downstream
+      // CREATE misses the case-preserved table on a case-folding engine
+      // (Snowflake). Capture the buildManifest Malloy is handed for the
+      // downstream to prove the seed was quoted for its connection's dialect
+      // before it becomes a FROM.
+      let seen:
+         | {
+              buildManifest?: {
+                 entries?: Record<string, { tableName?: string }>;
+              };
+           }
+         | undefined;
+      const runSQL = sinon.stub().resolves();
+      const connection = {
+         runSQL,
+         dialectName: "snowflake",
+      } as unknown as MalloyConnection;
+      const down = fakeSource({
+         name: "down",
+         sourceEntityId: "bdownaaaaaaaaaa",
+         dialectName: "snowflake",
+         onGetSQL: (o) => {
+            seen = o as typeof seen;
+         },
+      });
+      const compiled = compiledWith(
+         { down },
+         [["down"]],
+         new Map([["duckdb", connection]]),
+      );
+
+      await callExecute(
+         compiled,
+         [
+            {
+               sourceEntityId: "bdownaaaaaaaaaa",
+               materializedTableId: "mt-d",
+               physicalTableName: "down_v1",
+               realization: "COPY",
+            },
+         ],
+         {
+            up: {
+               sourceEntityId: "up",
+               physicalTableName: "schema.orders_base",
+               connectionName: "duckdb",
+            },
+         },
+      );
+
+      expect(seen?.buildManifest?.entries?.up?.tableName).toBe(
+         '"schema"."orders_base"',
+      );
+   });
+
+   it("seeds unquoted when the upstream's connection is absent from the build (older CP / cross-connection)", async () => {
+      // No connectionName on the seed (an older control plane) -> bind verbatim,
+      // exactly the pre-change behavior. Non-case-folding engines are unaffected;
+      // a case-folding engine self-signals by failing the downstream CREATE.
+      let seen:
+         | {
+              buildManifest?: {
+                 entries?: Record<string, { tableName?: string }>;
+              };
+           }
+         | undefined;
+      const runSQL = sinon.stub().resolves();
+      const connection = {
+         runSQL,
+         dialectName: "snowflake",
+      } as unknown as MalloyConnection;
+      const down = fakeSource({
+         name: "down",
+         sourceEntityId: "bdownaaaaaaaaaa",
+         dialectName: "snowflake",
+         onGetSQL: (o) => {
+            seen = o as typeof seen;
+         },
+      });
+      const compiled = compiledWith(
+         { down },
+         [["down"]],
+         new Map([["duckdb", connection]]),
+      );
+
+      await callExecute(
+         compiled,
+         [
+            {
+               sourceEntityId: "bdownaaaaaaaaaa",
+               materializedTableId: "mt-d",
+               physicalTableName: "down_v1",
+               realization: "COPY",
+            },
+         ],
+         { up: { sourceEntityId: "up", physicalTableName: "orders_base" } },
+      );
+
+      expect(seen?.buildManifest?.entries?.up?.tableName).toBe("orders_base");
+   });
+
    it("throws when an instructed graph's connection is missing", async () => {
       const s1 = fakeSource({ name: "s1", sourceEntityId: "b1aaaaaaaaaaaaaa" });
       const compiled = compiledWith({ s1 }, [["s1"]], new Map());
@@ -819,6 +941,7 @@ describe("buildOneSource", () => {
       connection: { runSQL: sinon.SinonStub },
       physicalTableName: string,
       dialectName?: string,
+      manifest: Manifest = new Manifest(),
    ): Promise<{ sourceEntityId: string; physicalTableName: string }> {
       const source = fakeSource({
          name: "orders",
@@ -847,7 +970,7 @@ describe("buildOneSource", () => {
          instruction,
          connection,
          { duckdb: "dig" },
-         new Manifest(),
+         manifest,
       );
    }
 
@@ -898,6 +1021,33 @@ describe("buildOneSource", () => {
       ]);
       expect(entry.physicalTableName).toBe("ds.orders_v1");
    });
+
+   it("records the QUOTED just-built name in the build manifest (matches the CREATE)", async () => {
+      // The build manifest feeds a downstream persist's FROM verbatim, so the
+      // recorded name must be the SAME quoted path the CREATE used — else a
+      // downstream built later in this run misses the case-preserved table on a
+      // case-folding engine. buildManifest also passes through Malloy's canonical
+      // path validation (requireCanonicalTablePathAnyDialect), so this proves the
+      // quoted form is accepted, not just produced.
+      const runSQL = sinon.stub().resolves();
+      const manifest = new Manifest();
+      await callBuildOneSource(
+         { runSQL },
+         "schema.orders_base",
+         "snowflake",
+         manifest,
+      );
+      expect(manifest.buildManifest.entries["abcdef1234567890"].tableName).toBe(
+         '"schema"."orders_base"',
+      );
+      // The logical (unquoted) name is what the CREATE was told to produce, and
+      // is exactly what quoteTablePath quoted for the DDL — read mirrors write.
+      const created = runSQL
+         .getCalls()
+         .map((c) => c.args[0] as string)
+         .find((s) => s.startsWith("CREATE TABLE"))!;
+      expect(created).toContain('"schema"."orders_base_abcdef123456"');
+   });
 });
 
 describe("runBuild (branch behavior)", () => {
@@ -922,7 +1072,11 @@ describe("runBuild (branch behavior)", () => {
             forceRefresh: boolean;
             buildInstructions: BuildInstruction[] | undefined;
             referenceManifest?:
-               | { sourceEntityId: string; physicalTableName: string }[]
+               | {
+                    sourceEntityId: string;
+                    physicalTableName: string;
+                    connectionName?: string;
+                 }[]
                | undefined;
             strictUpstreams?: boolean | undefined;
          },
@@ -989,7 +1143,11 @@ describe("runBuild (branch behavior)", () => {
             forceRefresh: false,
             buildInstructions: instructions,
             referenceManifest: [
-               { sourceEntityId: "up-1", physicalTableName: "upstream_tbl" },
+               {
+                  sourceEntityId: "up-1",
+                  physicalTableName: "upstream_tbl",
+                  connectionName: "sf",
+               },
             ],
             strictUpstreams: true,
          },
@@ -998,10 +1156,13 @@ describe("runBuild (branch behavior)", () => {
 
       // The reference manifest becomes the seed (carried) entries so a
       // downstream build resolves its upstream to the existing physical table.
+      // connectionName is carried through so the seed can be quoted for the
+      // upstream's dialect (see quoteSeedTablePath).
       expect(svc.executeInstructedBuild.firstCall.args[2]).toEqual({
          "up-1": {
             sourceEntityId: "up-1",
             physicalTableName: "upstream_tbl",
+            connectionName: "sf",
          },
       });
       // strictUpstreams flows through as the strict flag (5th arg).
@@ -1065,9 +1226,11 @@ describe("autoLoadManifest", () => {
       ).autoLoadManifest({ reloadAllModelsForPackage }, "pkg", entries);
 
       expect(reloadAllModelsForPackage.calledOnce).toBe(true);
+      // connectionName is carried alongside tableName so the bind step
+      // (Package.quoteBoundTableNames) can quote the path for its dialect.
       expect(reloadAllModelsForPackage.firstCall.args).toEqual([
          "pkg",
-         { b1: { tableName: "orders_v1" } },
+         { b1: { tableName: "orders_v1", connectionName: "duckdb" } },
       ]);
    });