@malloy-publisher/server 0.0.228 → 0.0.230

package/src/service/cron_evaluator.ts

@@ -0,0 +1,105 @@
+/**
+ * Evaluate a 5-field UNIX cron expression, in UTC, for the standalone
+ * materialization scheduler. Mirrors the control plane's `CronEvaluator`
+ * semantics (5-field, UTC, "strictly after") so a given `materialization.schedule`
+ * fires at the same instants whether the publisher runs standalone or the control
+ * plane drives it.
+ *
+ * Thin wrapper over `cron-parser` that pins three policies:
+ *   - **exactly 5 fields** — `cron-parser` also accepts an optional leading
+ *     seconds field (6 fields); the manifest contract is 5-field UNIX cron, so a
+ *     6-field expression is rejected rather than silently reinterpreted.
+ *   - **plain UNIX grammar** — `cron-parser` accepts extensions (`L` last,
+ *     `W` nearest-weekday, `#` nth-weekday, `?` Quartz no-specific) that the
+ *     control plane's `cron-utils` UNIX parser rejects. Without this pin, an
+ *     expression like `0 6 L * *` would validate and fire locally but, after
+ *     publish, silently never arm in production. Month/day names (JAN, MON-FRI)
+ *     are still allowed.
+ *   - **UTC** — schedules are absolute, never tied to the server's local zone.
+ */
+import { CronExpressionParser } from "cron-parser";
+
+/** Valid 3-letter month and day-of-week names in UNIX cron. */
+const UNIX_CRON_NAMES = new Set([
+   "JAN",
+   "FEB",
+   "MAR",
+   "APR",
+   "MAY",
+   "JUN",
+   "JUL",
+   "AUG",
+   "SEP",
+   "OCT",
+   "NOV",
+   "DEC",
+   "SUN",
+   "MON",
+   "TUE",
+   "WED",
+   "THU",
+   "FRI",
+   "SAT",
+]);
+
+export class CronEvaluator {
+   /** True when `expr` is a syntactically valid 5-field UNIX cron. */
+   isValid(expr: string): boolean {
+      if (!this.hasFiveFields(expr) || !this.isPlainUnixGrammar(expr)) {
+         return false;
+      }
+      try {
+         CronExpressionParser.parse(expr, { tz: "UTC" });
+         return true;
+      } catch {
+         return false;
+      }
+   }
+
+   /**
+    * The next fire instant strictly after `from` (UTC). Throws when `expr` is
+    * not a valid 5-field cron — callers guard with {@link isValid} (the
+    * scheduler skips + logs an invalid cron rather than letting it throw in the
+    * tick).
+    */
+   nextAfter(expr: string, from: Date): Date {
+      if (!this.hasFiveFields(expr) || !this.isPlainUnixGrammar(expr)) {
+         throw new Error(
+            `Expected a 5-field UNIX cron, got: ${JSON.stringify(expr)}`,
+         );
+      }
+      return CronExpressionParser.parse(expr, {
+         currentDate: from,
+         tz: "UTC",
+      })
+         .next()
+         .toDate();
+   }
+
+   /** Whitespace-split field count is exactly 5. */
+   private hasFiveFields(expr: string): boolean {
+      return typeof expr === "string" && expr.trim().split(/\s+/).length === 5;
+   }
+
+   /**
+    * Reject cron-parser / Quartz extensions absent from plain UNIX cron: `#`
+    * and `?` (never part of a name), and any alphabetic token that is not a
+    * UNIX month/day name — which catches `L`, `W`, `LW`, `L-3`, `15W`, etc.
+    * while still allowing named fields like `JUL` or `MON-FRI`.
+    */
+   private isPlainUnixGrammar(expr: string): boolean {
+      for (const field of expr.trim().split(/\s+/)) {
+         if (/[#?]/.test(field)) return false;
+         for (const token of field.split(/[,\-/]/)) {
+            if (token === "" || token === "*") continue;
+            if (
+               /[A-Za-z]/.test(token) &&
+               !UNIX_CRON_NAMES.has(token.toUpperCase())
+            ) {
+               return false;
+            }
+         }
+      }
+      return true;
+   }
+}

package/src/service/ducklake_lazy_attach.spec.ts

@@ -0,0 +1,110 @@
+import { DuckDBConnection } from "@malloydata/db-duckdb";
+import { afterEach, beforeEach, describe, expect, it } from "bun:test";
+import fs from "fs/promises";
+import path from "path";
+import sinon from "sinon";
+import { components } from "../api";
+import { buildEnvironmentMalloyConfig } from "./connection";
+
+type ApiConnection = components["schemas"]["Connection"];
+
+// A DuckLake connection whose Postgres catalog points at an unroutable host.
+// If anything on the boot path tried to reach the catalog, these tests would
+// hang or fail — which is precisely the regression they guard against.
+const UNREACHABLE_DUCKLAKE: ApiConnection = {
+   name: "ducklake_lazy",
+   type: "ducklake",
+   ducklakeConnection: {
+      catalog: {
+         postgresConnection: {
+            host: "192.0.2.1", // TEST-NET-1: guaranteed non-routable
+            port: 5432,
+            userName: "nobody",
+            password: "nobody",
+            databaseName: "catalog",
+         },
+      },
+      storage: {
+         bucketUrl: "s3://test-bucket",
+         s3Connection: {
+            accessKeyId: "test",
+            secretAccessKey: "test",
+         },
+      },
+   },
+} as ApiConnection;
+
+describe("DuckLake lazy attach", () => {
+   const envPath = path.join(process.cwd(), "test-ducklake-lazy-attach");
+
+   beforeEach(async () => {
+      await fs.mkdir(envPath, { recursive: true });
+   });
+
+   afterEach(async () => {
+      sinon.restore();
+      await fs.rm(envPath, { recursive: true, force: true }).catch(() => {});
+   });
+
+   it("does not attach the catalog while building the environment config (boot path)", async () => {
+      // Stub the DuckDB session so no real database or network work happens; the
+      // stub also lets us observe whether ANY SQL ran during config build.
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+
+      // Building the config is the worker boot path. It must be synchronous and
+      // must not open, construct, or touch the DuckLake session at all.
+      const config = buildEnvironmentMalloyConfig(
+         [UNREACHABLE_DUCKLAKE],
+         envPath,
+      );
+
+      expect(config.apiConnections.map((c) => c.name)).toContain(
+         "ducklake_lazy",
+      );
+      // The definitive assertion: zero SQL was issued building the config, so
+      // the (unreachable) catalog could not have been contacted on boot.
+      expect(runSQL.callCount).toBe(0);
+   });
+
+   it("cannot let an unavailable catalog block startup", async () => {
+      sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+
+      // Even with an unreachable catalog, constructing the environment config
+      // resolves promptly and never awaits the catalog. If attach were on the
+      // boot path, this would hang against 192.0.2.1 until a connect timeout.
+      const config = buildEnvironmentMalloyConfig(
+         [UNREACHABLE_DUCKLAKE],
+         envPath,
+      );
+      expect(config).toBeDefined();
+      expect(config.malloyConfig).toBeDefined();
+   });
+
+   it("attaches the catalog only on first connection lookup (serve path)", async () => {
+      const runSQL = sinon
+         .stub(DuckDBConnection.prototype, "runSQL")
+         .resolves({ rows: [] } as never);
+
+      const config = buildEnvironmentMalloyConfig(
+         [UNREACHABLE_DUCKLAKE],
+         envPath,
+      );
+      expect(runSQL.callCount).toBe(0);
+
+      // The lazy attach fires here, on the first lookup — not before.
+      await config.malloyConfig.connections.lookupConnection("ducklake_lazy");
+
+      const issued = runSQL.getCalls().map((c) => String(c.args[0]));
+      const attachedDuckLake = issued.some((sql) =>
+         sql.includes("ducklake:postgres:"),
+      );
+      expect(runSQL.callCount).toBeGreaterThan(0);
+      expect(attachedDuckLake).toBe(true);
+
+      await config.releaseConnections().catch(() => {});
+   });
+});

package/src/service/environment.ts

@@ -21,6 +21,7 @@ import {
    ServiceUnavailableError,
 } from "../errors";
 import { logger } from "../logger";
+import { redactPgSecrets } from "../pg_helpers";
 import { recordManifestBind } from "../materialization_metrics";
 import {
    assertSafeEnvironmentPath,
@@ -138,6 +139,27 @@ export class Environment {
    // AB/BA deadlock path.
    private packageMutexes = new Map<string, Mutex>();
    private packageStatuses: Map<string, PackageInfo> = new Map();
+   /**
+    * Configured packages that failed to load, keyed by name, with the reason.
+    *
+    * A load failure is not fatal: the package is omitted and its siblings serve
+    * on. It is also observable exactly once, because the failing load deletes
+    * the `packageStatuses` entry that `listPackages` enumerates, so the next
+    * listing no longer knows the package was ever configured. That makes this
+    * the only lasting record. Read by EnvironmentStore.getStatus.
+    */
+   private failedPackages: Map<string, string> = new Map();
+   /**
+    * Why a configured package never reached the disk, keyed by package name.
+    *
+    * Separate from {@link failedPackages} because it is the more specific
+    * answer and has to win. A package whose location failed to mount is still
+    * seeded SERVING by addEnvironment, so its later lazy load fails on the
+    * manifest that was never copied and `listPackages` records that instead.
+    * Reporting "Package manifest ... does not exist." for what was really a
+    * typo'd `location` sends the reader hunting in the wrong place.
+    */
+   private mountErrors: Map<string, string> = new Map();
    private malloyConfig: EnvironmentMalloyConfig;
    private connectionMutex = new Mutex();
    private retiredConnectionGenerations =
@@ -158,6 +180,11 @@ export class Environment {
       return this.environmentPath;
    }
 
+   /** This environment's name (the canonical key used by the API/service). */
+   public getEnvironmentName(): string {
+      return this.environmentName;
+   }
+
    constructor(
       environmentName: string,
       environmentPath: string,
@@ -456,10 +483,15 @@ export class Environment {
                );
             }
 
-            // Authorize backstop (the *who* axis, 403). Only run when the model
-            // declares gates so ungated compiles don't pay for the extra
-            // final-query compile.
-            if (queryMaterializer && gateModel?.hasAuthorize()) {
+            // Authorize backstop (the *who* axis, 403). NOT guarded by
+            // hasAuthorize(): that only inspects top-level modelDef.contents
+            // sources, so a gated source reached only via a cross-file/deep
+            // join is invisible to it and this backstop would silently never
+            // run for such a model — the same bypass assertAuthorizedForRunnable
+            // itself closes on the query path (see model.ts
+            // assertAuthorizedForAllSources). The own-source probe and joined-
+            // gate walk it runs are cheap no-ops for a genuinely ungated model.
+            if (queryMaterializer && gateModel) {
                await gateModel.assertAuthorizedForRunnable(
                   queryMaterializer,
                   givens ?? {},
@@ -589,6 +621,17 @@ export class Environment {
       );
    }
 
+   /**
+    * Snapshot of the packages currently loaded in memory (does not trigger a
+    * load or reload). Used by the standalone materialization scheduler to sweep
+    * only already-loaded packages — a not-yet-loaded package is simply not
+    * scheduled until something else loads it, so the scheduler never forces a
+    * load of its own.
+    */
+   public getLoadedPackages(): Package[] {
+      return [...this.packages.values()];
+   }
+
    public async listPackages(): Promise<ApiPackage[]> {
       logger.debug("Listing packages", {
          environmentPath: this.environmentPath,
@@ -613,6 +656,24 @@ export class Environment {
                   );
                   // Directory did not contain a valid package.json file -- therefore, it's not a package.
                   // Or it timed out
+                  // Redact before this reaches getStatus: compiling a model
+                  // resolves the package's connections, so a Postgres/DuckLake
+                  // ATTACH failure surfaces here carrying the connection string
+                  // (connection.ts builds it, and redacts it before logging for
+                  // the same reason). A log line was the old destination; this
+                  // one is an HTTP response body.
+                  //
+                  // Reduces the exposure, does not remove it: redactPgSecrets
+                  // only covers keyword-form `password=`, which is what
+                  // buildPgConnectionString emits. A URL-form connectionString
+                  // supplied verbatim in config still carries its credentials
+                  // through. Widen the helper rather than trusting this call.
+                  this.failedPackages.set(
+                     packageName,
+                     redactPgSecrets(
+                        error instanceof Error ? error.message : String(error),
+                     ),
+                  );
                   return undefined;
                }
             }),
@@ -876,6 +937,9 @@ export class Environment {
          }
          this.packages.set(packageName, _package);
          this.setPackageStatus(packageName, PackageStatus.SERVING);
+         // It loaded, so any earlier failure is stale. A package that failed at
+         // boot can be fixed on disk and reloaded without a restart.
+         this.clearPackageLoadFailure(packageName);
          logger.debug(`Successfully loaded package ${packageName}`);
 
          return _package;
@@ -957,6 +1021,10 @@ export class Environment {
          throw error;
       }
       this.setPackageStatus(packageName, PackageStatus.SERVING);
+      // Same reasoning as the load and install paths: it is serving now, so an
+      // earlier boot failure is stale. Without this, a package fixed on disk
+      // and re-added keeps its loadError for the life of the process.
+      this.clearPackageLoadFailure(packageName);
       return this.packages.get(packageName);
    }
 
@@ -1145,6 +1213,8 @@ export class Environment {
 
          this.packages.set(packageName, newPackage);
          this.setPackageStatus(packageName, PackageStatus.SERVING);
+         // Publishing a fixed package clears the boot failure it replaces.
+         this.clearPackageLoadFailure(packageName);
 
          if (oldPackage) {
             this.retireConnectionGeneration(`package ${packageName}`, () =>
@@ -1312,6 +1382,8 @@ export class Environment {
          explores?: string[];
          queryableSources?: "declared" | "all";
          manifestLocation?: string | null;
+         scope?: ApiPackage["scope"];
+         materialization?: ApiPackage["materialization"];
       },
    ): Promise<void> {
       const packagePath = safeJoinUnderRoot(this.environmentPath, packageName);
@@ -1344,6 +1416,10 @@ export class Environment {
             ...(metadata.manifestLocation !== undefined
                ? { manifestLocation: metadata.manifestLocation }
                : {}),
+            ...(metadata.scope !== undefined ? { scope: metadata.scope } : {}),
+            ...(metadata.materialization !== undefined
+               ? { materialization: metadata.materialization }
+               : {}),
          };
 
          // Write back to file
@@ -1396,6 +1472,29 @@ export class Environment {
             body.manifestLocation !== undefined
                ? body.manifestLocation
                : existing.manifestLocation;
+         // Persist `scope` and `materialization` (the schedule cron) are
+         // editable via the API — both are writable in the schema. When the
+         // body carries a value, apply it; otherwise preserve the
+         // manifest-derived one (a name/description-only PATCH must not wipe
+         // them, and the control plane must not misread the gap as a removal).
+         // Changing the schedule re-arms the standalone scheduler on its next
+         // tick — no reload needed.
+         //
+         // A *null* scope/materialization is treated the same as omitted
+         // (preserve), NOT a wipe: the control plane's post-build rebind PATCH
+         // carries only name/location/manifestLocation, but a client that
+         // serializes unset fields as explicit null must not thereby trip the
+         // policy gate below (a rejection there fails the orchestrated run) or
+         // reset the persisted policy. `manifestLocation` is deliberately
+         // different — null there means "clear" (revert to live), which the
+         // orchestrator relies on.
+         const scopeProvided = body.scope != null;
+         const materializationProvided = body.materialization != null;
+         const editingPolicy = scopeProvided || materializationProvided;
+         const scope = scopeProvided ? body.scope : existing.scope;
+         const materialization = materializationProvided
+            ? body.materialization
+            : existing.materialization;
          _package.setPackageMetadata({
             name: body.name,
             description: body.description,
@@ -1404,25 +1503,27 @@ export class Environment {
             explores,
             queryableSources,
             manifestLocation,
-            // Carry the manifest-derived materialization policy through the
-            // PATCH. `setPackageMetadata` replaces the whole object, so omitting
-            // this wipes the in-memory `materialization` until the next reload —
-            // which makes a later getPackage report no schedule and lets the
-            // control plane misread the gap as a schedule removal. It is not a
-            // PATCH-editable field, so always preserve the existing value.
-            materialization: existing.materialization,
-            // Same rationale for the manifest-derived persist `scope`: not
-            // PATCH-editable, so preserve it rather than dropping it to the
-            // default until the next reload.
-            scope: existing.scope,
+            materialization,
+            scope,
          });
 
          // Strict-reject, symmetric with the publish path
          // (package.controller.addPackage): validate the resulting explores
          // against the live model set and restore the prior metadata before
          // rejecting, so a bad update neither persists nor mutates the served
-         // surface.
-         const invalidMsg = _package.formatInvalidExplores();
+         // surface. When the body edits the persistence policy (scope /
+         // materialization), also enforce the same scope/schedule/freshness/cron
+         // rules a publish enforces — but only then, so a description-only PATCH
+         // on a package with a pre-existing (load-tolerated) policy warning is
+         // not newly rejected. Cron validity is one of these rules
+         // (persistencePolicyWarnings Rule 4), so publish, PATCH, load, and the
+         // scheduler all enforce it identically.
+         const policyMsg = editingPolicy
+            ? _package.formatInvalidPersistencePolicy()
+            : "";
+         const invalidMsg = [_package.formatInvalidExplores(), policyMsg]
+            .filter(Boolean)
+            .join("\n");
          if (invalidMsg) {
             _package.setPackageMetadata(existing);
             throw new BadRequestError(invalidMsg);
@@ -1434,6 +1535,13 @@ export class Environment {
             explores: normalizedExplores,
             queryableSources: body.queryableSources,
             manifestLocation: body.manifestLocation,
+            // Only write when explicitly provided (non-null): mirrors the
+            // null-as-absent rule above, so a rebind PATCH neither wipes the
+            // persisted policy nor writes a stray `scope: null`.
+            scope: scopeProvided ? body.scope : undefined,
+            materialization: materializationProvided
+               ? body.materialization
+               : undefined,
          });
 
          // When the body changes manifestLocation, apply it now so the new
@@ -1455,6 +1563,29 @@ export class Environment {
       return this.packageStatuses.get(packageName);
    }
 
+   /**
+    * Record why a configured package's location never mounted, so /status can
+    * name the real cause instead of the missing-manifest fallout it produces.
+    * Called by EnvironmentStore right after the environment is created.
+    */
+   public setPackageMountError(packageName: string, message: string): void {
+      this.mountErrors.set(packageName, message);
+   }
+
+   /** Forget any recorded failure for a package, whatever its cause. */
+   private clearPackageLoadFailure(packageName: string): void {
+      this.failedPackages.delete(packageName);
+      this.mountErrors.delete(packageName);
+   }
+
+   /** Packages configured for this environment that did not load, and why. */
+   public getFailedPackages(): ReadonlyMap<string, string> {
+      if (this.mountErrors.size === 0) return this.failedPackages;
+      // Mount errors last, so the specific cause overwrites the generic
+      // manifest error that the un-mounted package produces on its lazy load.
+      return new Map([...this.failedPackages, ...this.mountErrors]);
+   }
+
    public setPackageStatus(packageName: string, status: PackageStatus): void {
       const currentStatus = this.packageStatuses.get(packageName);
       this.packageStatuses.set(packageName, {
@@ -1471,6 +1602,14 @@ export class Environment {
    public async deletePackage(packageName: string): Promise<void> {
       assertSafePackageName(packageName);
       return this.withPackageLock(packageName, async () => {
+         // Clear the load failure before the early return, not after it. A
+         // package that failed to load is not in `packages` (the load catch
+         // evicts it), so deleting it takes the early return every time, while
+         // the controller still drops its config row. Leaving the entry would
+         // make getStatus report a loadError for a package that is no longer
+         // configured, which is the one thing that channel must not do.
+         this.clearPackageLoadFailure(packageName);
+
          const _package = this.packages.get(packageName);
          if (!_package) {
             return;