@malloy-publisher/server 0.0.227 → 0.0.229

package/src/service/cron_evaluator.spec.ts

@@ -0,0 +1,79 @@
+import { describe, expect, it } from "bun:test";
+import { CronEvaluator } from "./cron_evaluator";
+
+describe("CronEvaluator", () => {
+   const cron = new CronEvaluator();
+
+   describe("isValid", () => {
+      it("accepts 5-field UNIX crons", () => {
+         expect(cron.isValid("0 6 * * *")).toBe(true);
+         expect(cron.isValid("*/15 * * * *")).toBe(true);
+         expect(cron.isValid("0 0 1 1 *")).toBe(true);
+      });
+
+      it("rejects a 6-field (seconds) cron — the contract is 5-field", () => {
+         expect(cron.isValid("0 0 6 * * *")).toBe(false);
+      });
+
+      it("rejects garbage and empty input", () => {
+         expect(cron.isValid("not a cron")).toBe(false);
+         expect(cron.isValid("")).toBe(false);
+         expect(cron.isValid("* * *")).toBe(false);
+         // Out-of-range field.
+         expect(cron.isValid("99 * * * *")).toBe(false);
+      });
+
+      it("rejects cron-parser / Quartz extensions the UNIX parser lacks", () => {
+         // These parse under cron-parser but the control plane's cron-utils
+         // UNIX parser rejects them — pinning the grammar keeps a cron from
+         // validating locally yet silently never arming in production.
+         expect(cron.isValid("0 6 L * *")).toBe(false); // last day of month
+         expect(cron.isValid("0 6 15W * *")).toBe(false); // nearest weekday
+         expect(cron.isValid("0 6 * * 1#2")).toBe(false); // 2nd Monday
+         expect(cron.isValid("0 0 ? * *")).toBe(false); // Quartz no-specific
+         expect(cron.isValid("0 6 LW * *")).toBe(false); // last weekday
+      });
+
+      it("accepts UNIX month/day names (not mistaken for extensions)", () => {
+         expect(cron.isValid("0 6 * JUL *")).toBe(true); // JUL contains 'L'
+         expect(cron.isValid("0 6 * * WED")).toBe(true); // WED contains 'W'
+         expect(cron.isValid("0 6 * * THU")).toBe(true); // THU contains 'H'
+         expect(cron.isValid("0 6 * * MON-FRI")).toBe(true);
+      });
+   });
+
+   describe("nextAfter (UTC, strictly after)", () => {
+      it("returns the next daily fire", () => {
+         const from = new Date("2026-07-13T00:00:00Z");
+         expect(cron.nextAfter("0 6 * * *", from).toISOString()).toBe(
+            "2026-07-13T06:00:00.000Z",
+         );
+      });
+
+      it("rolls to the next day when the fire time has passed", () => {
+         const from = new Date("2026-07-13T07:00:00Z");
+         expect(cron.nextAfter("0 6 * * *", from).toISOString()).toBe(
+            "2026-07-14T06:00:00.000Z",
+         );
+      });
+
+      it("advances strictly past a `from` that lands exactly on a fire", () => {
+         const from = new Date("2026-07-13T06:00:00Z");
+         // Strictly after: the 06:00 instant itself is excluded.
+         expect(cron.nextAfter("0 6 * * *", from).toISOString()).toBe(
+            "2026-07-14T06:00:00.000Z",
+         );
+      });
+
+      it("steps every 15 minutes", () => {
+         const from = new Date("2026-07-13T09:07:00Z");
+         expect(cron.nextAfter("*/15 * * * *", from).toISOString()).toBe(
+            "2026-07-13T09:15:00.000Z",
+         );
+      });
+
+      it("throws on an invalid cron (callers guard with isValid)", () => {
+         expect(() => cron.nextAfter("nope", new Date())).toThrow();
+      });
+   });
+});

package/src/service/cron_evaluator.ts

@@ -0,0 +1,105 @@
+/**
+ * Evaluate a 5-field UNIX cron expression, in UTC, for the standalone
+ * materialization scheduler. Mirrors the control plane's `CronEvaluator`
+ * semantics (5-field, UTC, "strictly after") so a given `materialization.schedule`
+ * fires at the same instants whether the publisher runs standalone or the control
+ * plane drives it.
+ *
+ * Thin wrapper over `cron-parser` that pins three policies:
+ *   - **exactly 5 fields** — `cron-parser` also accepts an optional leading
+ *     seconds field (6 fields); the manifest contract is 5-field UNIX cron, so a
+ *     6-field expression is rejected rather than silently reinterpreted.
+ *   - **plain UNIX grammar** — `cron-parser` accepts extensions (`L` last,
+ *     `W` nearest-weekday, `#` nth-weekday, `?` Quartz no-specific) that the
+ *     control plane's `cron-utils` UNIX parser rejects. Without this pin, an
+ *     expression like `0 6 L * *` would validate and fire locally but, after
+ *     publish, silently never arm in production. Month/day names (JAN, MON-FRI)
+ *     are still allowed.
+ *   - **UTC** — schedules are absolute, never tied to the server's local zone.
+ */
+import { CronExpressionParser } from "cron-parser";
+
+/** Valid 3-letter month and day-of-week names in UNIX cron. */
+const UNIX_CRON_NAMES = new Set([
+   "JAN",
+   "FEB",
+   "MAR",
+   "APR",
+   "MAY",
+   "JUN",
+   "JUL",
+   "AUG",
+   "SEP",
+   "OCT",
+   "NOV",
+   "DEC",
+   "SUN",
+   "MON",
+   "TUE",
+   "WED",
+   "THU",
+   "FRI",
+   "SAT",
+]);
+
+export class CronEvaluator {
+   /** True when `expr` is a syntactically valid 5-field UNIX cron. */
+   isValid(expr: string): boolean {
+      if (!this.hasFiveFields(expr) || !this.isPlainUnixGrammar(expr)) {
+         return false;
+      }
+      try {
+         CronExpressionParser.parse(expr, { tz: "UTC" });
+         return true;
+      } catch {
+         return false;
+      }
+   }
+
+   /**
+    * The next fire instant strictly after `from` (UTC). Throws when `expr` is
+    * not a valid 5-field cron — callers guard with {@link isValid} (the
+    * scheduler skips + logs an invalid cron rather than letting it throw in the
+    * tick).
+    */
+   nextAfter(expr: string, from: Date): Date {
+      if (!this.hasFiveFields(expr) || !this.isPlainUnixGrammar(expr)) {
+         throw new Error(
+            `Expected a 5-field UNIX cron, got: ${JSON.stringify(expr)}`,
+         );
+      }
+      return CronExpressionParser.parse(expr, {
+         currentDate: from,
+         tz: "UTC",
+      })
+         .next()
+         .toDate();
+   }
+
+   /** Whitespace-split field count is exactly 5. */
+   private hasFiveFields(expr: string): boolean {
+      return typeof expr === "string" && expr.trim().split(/\s+/).length === 5;
+   }
+
+   /**
+    * Reject cron-parser / Quartz extensions absent from plain UNIX cron: `#`
+    * and `?` (never part of a name), and any alphabetic token that is not a
+    * UNIX month/day name — which catches `L`, `W`, `LW`, `L-3`, `15W`, etc.
+    * while still allowing named fields like `JUL` or `MON-FRI`.
+    */
+   private isPlainUnixGrammar(expr: string): boolean {
+      for (const field of expr.trim().split(/\s+/)) {
+         if (/[#?]/.test(field)) return false;
+         for (const token of field.split(/[,\-/]/)) {
+            if (token === "" || token === "*") continue;
+            if (
+               /[A-Za-z]/.test(token) &&
+               !UNIX_CRON_NAMES.has(token.toUpperCase())
+            ) {
+               return false;
+            }
+         }
+      }
+      return true;
+   }
+}

package/src/service/environment.ts

@@ -21,6 +21,7 @@ import {
    ServiceUnavailableError,
 } from "../errors";
 import { logger } from "../logger";
+import { redactPgSecrets } from "../pg_helpers";
 import { recordManifestBind } from "../materialization_metrics";
 import {
    assertSafeEnvironmentPath,
@@ -138,6 +139,16 @@ export class Environment {
    // AB/BA deadlock path.
    private packageMutexes = new Map<string, Mutex>();
    private packageStatuses: Map<string, PackageInfo> = new Map();
+   /**
+    * Configured packages that failed to load, keyed by name, with the reason.
+    *
+    * A load failure is not fatal: the package is omitted and its siblings serve
+    * on. It is also observable exactly once, because the failing load deletes
+    * the `packageStatuses` entry that `listPackages` enumerates, so the next
+    * listing no longer knows the package was ever configured. That makes this
+    * the only lasting record. Read by EnvironmentStore.getStatus.
+    */
+   private failedPackages: Map<string, string> = new Map();
    private malloyConfig: EnvironmentMalloyConfig;
    private connectionMutex = new Mutex();
    private retiredConnectionGenerations =
@@ -158,6 +169,11 @@ export class Environment {
       return this.environmentPath;
    }
 
+   /** This environment's name (the canonical key used by the API/service). */
+   public getEnvironmentName(): string {
+      return this.environmentName;
+   }
+
    constructor(
       environmentName: string,
       environmentPath: string,
@@ -471,9 +487,30 @@ export class Environment {
             if (includeSql && queryMaterializer) {
                try {
                   sql = await queryMaterializer.getSQL({ givens });
-               } catch {
-                  // Source may not contain a runnable query (e.g. only source definitions),
-                  // in which case we simply omit the sql field.
+               } catch (error) {
+                  // A bad caller given (unknown name, wrong-typed value, finalized
+                  // override, ...) surfaces as a Malloy `runtime-given-*` error.
+                  // Map it to a 400 rather than silently omitting `sql` (which is
+                  // indistinguishable from "no runnable query"). Duck-type on
+                  // `.code`; let a MalloyError fall to the outer catch → problems.
+                  // The `runtime-given-` prefix is pinned to Malloy's error codes
+                  // (given_binding.ts / runtime.ts, same as model.ts) — if they're
+                  // renamed upstream a bad given would silently revert to the omit
+                  // branch below, so keep the two in sync.
+                  const givenCode = (error as { code?: string })?.code;
+                  if (
+                     typeof givenCode === "string" &&
+                     givenCode.startsWith("runtime-given-")
+                  ) {
+                     throw new BadRequestError(
+                        error instanceof Error ? error.message : String(error),
+                     );
+                  }
+                  if (error instanceof MalloyError) {
+                     throw error;
+                  }
+                  // Otherwise the source may just not contain a runnable query
+                  // (e.g. only source definitions) — omit the sql field.
                }
             }
 
@@ -568,6 +605,17 @@ export class Environment {
       );
    }
 
+   /**
+    * Snapshot of the packages currently loaded in memory (does not trigger a
+    * load or reload). Used by the standalone materialization scheduler to sweep
+    * only already-loaded packages — a not-yet-loaded package is simply not
+    * scheduled until something else loads it, so the scheduler never forces a
+    * load of its own.
+    */
+   public getLoadedPackages(): Package[] {
+      return [...this.packages.values()];
+   }
+
    public async listPackages(): Promise<ApiPackage[]> {
       logger.debug("Listing packages", {
          environmentPath: this.environmentPath,
@@ -592,6 +640,24 @@ export class Environment {
                   );
                   // Directory did not contain a valid package.json file -- therefore, it's not a package.
                   // Or it timed out
+                  // Redact before this reaches getStatus: compiling a model
+                  // resolves the package's connections, so a Postgres/DuckLake
+                  // ATTACH failure surfaces here carrying the connection string
+                  // (connection.ts builds it, and redacts it before logging for
+                  // the same reason). A log line was the old destination; this
+                  // one is an HTTP response body.
+                  //
+                  // Reduces the exposure, does not remove it: redactPgSecrets
+                  // only covers keyword-form `password=`, which is what
+                  // buildPgConnectionString emits. A URL-form connectionString
+                  // supplied verbatim in config still carries its credentials
+                  // through. Widen the helper rather than trusting this call.
+                  this.failedPackages.set(
+                     packageName,
+                     redactPgSecrets(
+                        error instanceof Error ? error.message : String(error),
+                     ),
+                  );
                   return undefined;
                }
             }),
@@ -855,13 +921,25 @@ export class Environment {
          }
          this.packages.set(packageName, _package);
          this.setPackageStatus(packageName, PackageStatus.SERVING);
+         // It loaded, so any earlier failure is stale. A package that failed at
+         // boot can be fixed on disk and reloaded without a restart.
+         this.failedPackages.delete(packageName);
          logger.debug(`Successfully loaded package ${packageName}`);
 
          return _package;
       } catch (error) {
          logger.error(`Failed to load package ${packageName}`, { error });
-         this.packages.delete(packageName);
-         this.packageStatuses.delete(packageName);
+         if (existingPackage !== undefined && reload) {
+            // A failed RELOAD must not take down a package that is already
+            // serving. The compiled model in `packages` is still the last good
+            // one (it is only replaced on success), so keep serving it and let
+            // the caller surface the error instead of evicting the package and
+            // leaving the environment with nothing to answer from.
+            this.setPackageStatus(packageName, PackageStatus.SERVING);
+         } else {
+            this.packages.delete(packageName);
+            this.packageStatuses.delete(packageName);
+         }
          throw error;
       }
    }
@@ -927,6 +1005,10 @@ export class Environment {
          throw error;
       }
       this.setPackageStatus(packageName, PackageStatus.SERVING);
+      // Same reasoning as the load and install paths: it is serving now, so an
+      // earlier boot failure is stale. Without this, a package fixed on disk
+      // and re-added keeps its loadError for the life of the process.
+      this.failedPackages.delete(packageName);
       return this.packages.get(packageName);
    }
 
@@ -1019,6 +1101,10 @@ export class Environment {
                packageName,
                canonicalPath,
                () => this.malloyConfig.malloyConfig,
+               // This tree was just staged into place by the rename above, so a
+               // failed load leaves a half-built directory that is ours to
+               // remove; the rollback below restores the previous one.
+               true,
             );
             // Strict-reject hook (publish/update only — reload passes no
             // validator and stays fail-safe). Throw INSIDE the try so the
@@ -1064,7 +1150,37 @@ export class Environment {
             await fs.promises
                .rm(stagingPath, { recursive: true, force: true })
                .catch(() => {});
-            this.deletePackageStatus(packageName);
+            if (oldPackage && restored) {
+               // The rollback put the old tree back and the previous package is
+               // still in `this.packages` (it is only replaced on success
+               // below), so it is genuinely still serving. Deleting its status
+               // would strand it: listPackages enumerates packageStatuses, so
+               // the package would answer getPackage while being invisible to
+               // listings and discovery until a restart.
+               this.setPackageStatus(packageName, PackageStatus.SERVING);
+            } else {
+               // Either there was nothing to fall back to (a first install), or
+               // the restore did not happen: the rename-back threw, or the old
+               // tree was never on disk to retire. The canonical path is then
+               // missing or still holds the rejected content, so the cached
+               // package no longer matches disk and must not be advertised as
+               // serving. Drop both, and keep the two maps agreeing.
+               this.deletePackageStatus(packageName);
+               if (oldPackage) {
+                  // Retire before dropping it, the same way every other eviction
+                  // here does: once it leaves this.packages, closeAllConnections
+                  // can no longer reach its MalloyConfig, so its native handles
+                  // would never be released. Retire rather than shut down
+                  // inline, because withPackageLock does not cover queries that
+                  // already took this Package from an earlier getPackage; the
+                  // drain lets those finish first.
+                  this.retireConnectionGeneration(
+                     `package ${packageName}`,
+                     () => oldPackage.getMalloyConfig().shutdown("close"),
+                  );
+               }
+               this.packages.delete(packageName);
+            }
             logger.debug("install.phase2.rollback", {
                environmentName: this.environmentName,
                packageName,
@@ -1081,6 +1197,8 @@ export class Environment {
 
          this.packages.set(packageName, newPackage);
          this.setPackageStatus(packageName, PackageStatus.SERVING);
+         // Publishing a fixed package clears the boot failure it replaces.
+         this.failedPackages.delete(packageName);
 
          if (oldPackage) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
@@ -1248,6 +1366,8 @@ export class Environment {
          explores?: string[];
          queryableSources?: "declared" | "all";
          manifestLocation?: string | null;
+         scope?: ApiPackage["scope"];
+         materialization?: ApiPackage["materialization"];
       },
    ): Promise<void> {
       const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
@@ -1280,6 +1400,10 @@ export class Environment {
             ...(metadata.manifestLocation !== undefined
                ? { manifestLocation: metadata.manifestLocation }
                : {}),
+            ...(metadata.scope !== undefined ? { scope: metadata.scope } : {}),
+            ...(metadata.materialization !== undefined
+               ? { materialization: metadata.materialization }
+               : {}),
          };
 
          // Write back to file
@@ -1332,6 +1456,29 @@ export class Environment {
             body.manifestLocation !== undefined
                ? body.manifestLocation
                : existing.manifestLocation;
+         // Persist `scope` and `materialization` (the schedule cron) are
+         // editable via the API — both are writable in the schema. When the
+         // body carries a value, apply it; otherwise preserve the
+         // manifest-derived one (a name/description-only PATCH must not wipe
+         // them, and the control plane must not misread the gap as a removal).
+         // Changing the schedule re-arms the standalone scheduler on its next
+         // tick — no reload needed.
+         //
+         // A *null* scope/materialization is treated the same as omitted
+         // (preserve), NOT a wipe: the control plane's post-build rebind PATCH
+         // carries only name/location/manifestLocation, but a client that
+         // serializes unset fields as explicit null must not thereby trip the
+         // policy gate below (a rejection there fails the orchestrated run) or
+         // reset the persisted policy. `manifestLocation` is deliberately
+         // different — null there means "clear" (revert to live), which the
+         // orchestrator relies on.
+         const scopeProvided = body.scope != null;
+         const materializationProvided = body.materialization != null;
+         const editingPolicy = scopeProvided || materializationProvided;
+         const scope = scopeProvided ? body.scope : existing.scope;
+         const materialization = materializationProvided
+            ? body.materialization
+            : existing.materialization;
          _package.setPackageMetadata({
             name: body.name,
             description: body.description,
@@ -1340,25 +1487,27 @@ export class Environment {
             explores,
             queryableSources,
             manifestLocation,
-            // Carry the manifest-derived materialization policy through the
-            // PATCH. `setPackageMetadata` replaces the whole object, so omitting
-            // this wipes the in-memory `materialization` until the next reload —
-            // which makes a later getPackage report no schedule and lets the
-            // control plane misread the gap as a schedule removal. It is not a
-            // PATCH-editable field, so always preserve the existing value.
-            materialization: existing.materialization,
-            // Same rationale for the manifest-derived persist `scope`: not
-            // PATCH-editable, so preserve it rather than dropping it to the
-            // default until the next reload.
-            scope: existing.scope,
+            materialization,
+            scope,
          });
 
          // Strict-reject, symmetric with the publish path
          // (package.controller.addPackage): validate the resulting explores
          // against the live model set and restore the prior metadata before
          // rejecting, so a bad update neither persists nor mutates the served
-         // surface.
-         const invalidMsg = _package.formatInvalidExplores();
+         // surface. When the body edits the persistence policy (scope /
+         // materialization), also enforce the same scope/schedule/freshness/cron
+         // rules a publish enforces — but only then, so a description-only PATCH
+         // on a package with a pre-existing (load-tolerated) policy warning is
+         // not newly rejected. Cron validity is one of these rules
+         // (persistencePolicyWarnings Rule 4), so publish, PATCH, load, and the
+         // scheduler all enforce it identically.
+         const policyMsg = editingPolicy
+            ? _package.formatInvalidPersistencePolicy()
+            : "";
+         const invalidMsg = [_package.formatInvalidExplores(), policyMsg]
+            .filter(Boolean)
+            .join("\n");
          if (invalidMsg) {
             _package.setPackageMetadata(existing);
             throw new BadRequestError(invalidMsg);
@@ -1370,6 +1519,13 @@ export class Environment {
             explores: normalizedExplores,
             queryableSources: body.queryableSources,
             manifestLocation: body.manifestLocation,
+            // Only write when explicitly provided (non-null): mirrors the
+            // null-as-absent rule above, so a rebind PATCH neither wipes the
+            // persisted policy nor writes a stray `scope: null`.
+            scope: scopeProvided ? body.scope : undefined,
+            materialization: materializationProvided
+               ? body.materialization
+               : undefined,
          });
 
          // When the body changes manifestLocation, apply it now so the new
@@ -1391,6 +1547,11 @@ export class Environment {
       return this.packageStatuses.get(packageName);
    }
 
+   /** Packages configured for this environment that did not load, and why. */
+   public getFailedPackages(): ReadonlyMap<string, string> {
+      return this.failedPackages;
+   }
+
    public setPackageStatus(packageName: string, status: PackageStatus): void {
       const currentStatus = this.packageStatuses.get(packageName);
       this.packageStatuses.set(packageName, {
@@ -1407,6 +1568,14 @@ export class Environment {
    public async deletePackage(packageName: string): Promise<void> {
       assertSafePackageName(packageName);
       return this.withPackageLock(packageName, async () => {
+         // Clear the load failure before the early return, not after it. A
+         // package that failed to load is not in `packages` (the load catch
+         // evicts it), so deleting it takes the early return every time, while
+         // the controller still drops its config row. Leaving the entry would
+         // make getStatus report a loadError for a package that is no longer
+         // configured, which is the one thing that channel must not do.
+         this.failedPackages.delete(packageName);
+
          const _package = this.packages.get(packageName);
          if (!_package) {
             return;