@malloy-publisher/server 0.0.227 → 0.0.229

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.227",
+   "version": "0.0.229",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -59,6 +59,7 @@
       "body-parser": "^1.20.2",
       "chokidar": "^4.0.3",
       "cors": "^2.8.5",
+      "cron-parser": "^5.6.2",
       "express": "^4.21.0",
       "extract-zip": "^2.0.1",
       "globals": "^15.9.0",
@@ -69,7 +70,8 @@
       "node-cron": "^3.0.3",
       "recursive-readdir": "^2.2.3",
       "simple-git": "^3.28.0",
-      "uuid": "^11.0.3"
+      "uuid": "^11.0.3",
+      "winston": "^3.17.0"
    },
    "devDependencies": {
       "@eslint/compat": "^1.2.7",

package/publisher.config.example.bigquery.json

@@ -2,19 +2,19 @@
   "frozenConfig": false,
   "environments": [
     {
-      "name": "malloy-samples",
+      "name": "examples",
       "packages": [
         {
-          "name": "ecommerce",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+          "name": "storefront",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
         },
         {
-          "name": "imdb",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+          "name": "governed-analytics",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
         },
         {
-          "name": "faa",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+          "name": "html-data-app",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
         },
         {
           "name": "bigquery-hackernews",

package/publisher.config.example.duckdb.json

@@ -2,19 +2,19 @@
   "frozenConfig": false,
   "environments": [
     {
-      "name": "malloy-samples",
+      "name": "examples",
       "packages": [
         {
-          "name": "ecommerce",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+          "name": "storefront",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
         },
         {
-          "name": "imdb",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+          "name": "governed-analytics",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
         },
         {
-          "name": "faa",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+          "name": "html-data-app",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
         }
       ],
       "connections": []

package/publisher.config.json

@@ -2,23 +2,19 @@
   "frozenConfig": false,
   "environments": [
     {
-      "name": "malloy-samples",
+      "name": "examples",
       "packages": [
         {
-          "name": "ecommerce",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+          "name": "storefront",
+          "location": "../../examples/storefront"
         },
         {
-          "name": "imdb",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+          "name": "governed-analytics",
+          "location": "../../examples/governed-analytics"
         },
         {
-          "name": "faa",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
-        },
-        {
-          "name": "faa-givens-demo",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa-givens-demo"
+          "name": "html-data-app",
+          "location": "../../examples/html-data-app"
         }
       ],
       "connections": []

package/src/config.spec.ts

@@ -1,7 +1,11 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import fs from "fs";
 import path from "path";
-import { getPublisherConfig, type PublisherConfig } from "./config";
+import {
+   getPublisherConfig,
+   getPublisherConfigDir,
+   type PublisherConfig,
+} from "./config";
 import { PUBLISHER_CONFIG_NAME } from "./constants";
 
 describe("Config Environment Variable Substitution", () => {
@@ -1040,8 +1044,8 @@ describe("Config path resolution (--config and bundled default)", () => {
 
       expect(result.environments.length).toBeGreaterThan(0);
       const env = result.environments[0];
-      expect(env.name).toBe("malloy-samples");
-      expect(env.packages.some((p) => p.name === "ecommerce")).toBe(true);
+      expect(env.name).toBe("examples");
+      expect(env.packages.some((p) => p.name === "storefront")).toBe(true);
       expect(env.packages.some((p) => p.name === "bigquery-hackernews")).toBe(
          false,
       );
@@ -1408,3 +1412,85 @@ describe("getMaxConcurrentQueries", () => {
       expect(() => getMaxConcurrentQueries()).toThrow();
    });
 });
+
+describe("getPublisherConfigDir", () => {
+   const testRoot = path.join(process.cwd(), "test-temp-config-dir");
+   const elsewhere = path.join(testRoot, "elsewhere");
+   const savedConfigPath = process.env.PUBLISHER_CONFIG_PATH;
+
+   beforeEach(() => {
+      fs.mkdirSync(elsewhere, { recursive: true });
+      // Both knobs feed resolvePublisherConfigPath, and every spec file shares
+      // one process, so clear both rather than inherit whatever ran before.
+      delete process.env.PUBLISHER_CONFIG_PATH;
+      delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
+   });
+
+   afterEach(() => {
+      fs.rmSync(testRoot, { recursive: true, force: true });
+      // Clear both, not just the one this describe saves: every spec file shares
+      // one process, so a knob left set here follows whatever runs next.
+      delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
+      if (savedConfigPath === undefined) {
+         delete process.env.PUBLISHER_CONFIG_PATH;
+      } else {
+         process.env.PUBLISHER_CONFIG_PATH = savedConfigPath;
+      }
+   });
+
+   it("returns the server root when the config sits there", () => {
+      fs.writeFileSync(
+         path.join(testRoot, PUBLISHER_CONFIG_NAME),
+         JSON.stringify({ environments: [] }),
+      );
+      expect(getPublisherConfigDir(testRoot)).toBe(testRoot);
+   });
+
+   it("follows PUBLISHER_CONFIG_PATH to a config outside the server root", () => {
+      // The --config case: a relative package location must anchor next to
+      // the config that declares it, not at whatever cwd the server booted in.
+      const configPath = path.join(elsewhere, PUBLISHER_CONFIG_NAME);
+      fs.writeFileSync(configPath, JSON.stringify({ environments: [] }));
+      process.env.PUBLISHER_CONFIG_PATH = configPath;
+      expect(getPublisherConfigDir(testRoot)).toBe(elsewhere);
+   });
+
+   it("returns null when no config resolves, so callers fall back to the server root", () => {
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+
+   it("returns an absolute anchor even when --config is relative", () => {
+      // An anchor that is itself relative would re-resolve against the cwd of
+      // whoever reads it, which is the bug this whole change is fixing.
+      const rel = path.relative(
+         process.cwd(),
+         path.join(elsewhere, PUBLISHER_CONFIG_NAME),
+      );
+      fs.writeFileSync(
+         path.join(elsewhere, PUBLISHER_CONFIG_NAME),
+         JSON.stringify({ environments: [] }),
+      );
+      process.env.PUBLISHER_CONFIG_PATH = rel;
+      const anchor = getPublisherConfigDir(testRoot);
+      expect(anchor).not.toBeNull();
+      expect(path.isAbsolute(anchor as string)).toBe(true);
+      expect(anchor).toBe(elsewhere);
+   });
+
+   it("returns null when --config names a directory instead of a file", () => {
+      // existsSync is true for a directory, so without a file check the anchor
+      // would silently become that directory's PARENT, for a config that never
+      // loads (getPublisherConfig catches the read error and returns empty).
+      process.env.PUBLISHER_CONFIG_PATH = elsewhere;
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+
+   it("returns null for the bundled default rather than anchoring inside the install", () => {
+      // Zero-arg `npx @malloy-publisher/server` resolves the config that ships
+      // inside the package. Anchoring a user's relative location under
+      // node_modules would be meaningless, so the caller falls back to the
+      // server root, which is what this did before the anchor moved.
+      process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+});

package/src/config.ts

@@ -71,7 +71,13 @@ function resolvePublisherConfigPath(serverRoot: string): {
    return null;
 }
 
-type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
+// Home paths are POSIX form only: `~/` expands, while bare `~`, `~user/…`,
+// and Windows-style `~\` are not local paths and are rejected downstream.
+type FilesystemPath =
+   | `./${string}`
+   | `../${string}`
+   | `/${string}`
+   | `~/${string}`;
 type GcsPath = `gs://${string}`;
 type ApiConnection = components["schemas"]["Connection"];
 export type Theme = components["schemas"]["Theme"];
@@ -256,6 +262,72 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
    };
 };
 
+/**
+ * Tunables for the standalone {@link MaterializationScheduler}. Sourced from
+ * environment variables at startup; see {@link getMaterializationSchedulerConfig}.
+ *
+ * The scheduler fires a package's `materialization.schedule` cron in **standalone**
+ * deployments (no control plane). It is **disabled by default** — an orchestrated
+ * deployment, whose control plane already drives materialization, never sets the
+ * enable flag, so the scheduler is never constructed there.
+ */
+export interface MaterializationSchedulerConfig {
+   /** Cadence of the due-schedule sweep, in milliseconds. */
+   tickIntervalMs: number;
+   /** Max packages fired per tick — a stampede guard for large deployments. */
+   maxFiresPerTick: number;
+}
+
+const DEFAULT_SCHEDULER_INTERVAL_MS = 60_000;
+const MIN_SCHEDULER_INTERVAL_MS = 1_000;
+const DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK = 10;
+
+/**
+ * Parse standalone-scheduler settings and return a validated config, or `null`
+ * when the feature is disabled. Disabled iff
+ * `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is unset/false — the default — so
+ * the scheduler never runs in an orchestrated deployment (where the control
+ * plane drives materialization itself).
+ *
+ * **Never set `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` on an orchestrated
+ * worker.** It is the primary safety guard: a control-plane-loaded package that
+ * is serving live has `manifestLocation === null`, so the scheduler's
+ * per-package `manifestLocation` skip does not cover it — only this flag being
+ * off keeps the standalone scheduler from double-driving the control plane.
+ *
+ * Throws at startup on malformed input so a typo surfaces loudly rather than
+ * silently disabling scheduling.
+ */
+export const getMaterializationSchedulerConfig =
+   (): MaterializationSchedulerConfig | null => {
+      const enabled =
+         parseBoolEnv("PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER") ?? false;
+      if (!enabled) {
+         return null;
+      }
+
+      const tickIntervalMs =
+         parseIntEnv("PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS") ??
+         DEFAULT_SCHEDULER_INTERVAL_MS;
+      const maxFiresPerTick =
+         parseIntEnv(
+            "PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK",
+         ) ?? DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK;
+
+      if (tickIntervalMs < MIN_SCHEDULER_INTERVAL_MS) {
+         throw new Error(
+            `PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS must be >= ${MIN_SCHEDULER_INTERVAL_MS} (got ${tickIntervalMs})`,
+         );
+      }
+      if (maxFiresPerTick <= 0) {
+         throw new Error(
+            `PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK must be a positive integer (got ${maxFiresPerTick})`,
+         );
+      }
+
+      return { tickIntervalMs, maxFiresPerTick };
+   };
+
 /**
  * Resolve the row cap applied to ad-hoc connection SQL queries.
  * Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
@@ -404,6 +476,35 @@ function processConfigValue(value: unknown): unknown {
    return value;
 }
 
+/**
+ * Absolute directory that a relative package `location` is resolved against:
+ * the one holding the active config file.
+ *
+ * Null when there is nothing sensible to anchor to, leaving the caller to pick
+ * a base. That covers three cases: no config at all; the bundled default, which
+ * lives inside the installed package, where anchoring a user's relative path
+ * somewhere under node_modules would be meaningless (and which declares only
+ * remote locations of its own); and a `--config` that names a directory rather
+ * than a file, which cannot be read as a config, so its parent is not an anchor
+ * anyone asked for.
+ */
+export const getPublisherConfigDir = (serverRoot: string): string | null => {
+   const resolved = resolvePublisherConfigPath(serverRoot);
+   if (!resolved || resolved.isBundledDefault) {
+      return null;
+   }
+   try {
+      if (!fs.statSync(resolved.path).isFile()) {
+         return null;
+      }
+   } catch {
+      return null;
+   }
+   // Resolve: `--config` may be relative, and an anchor that is itself relative
+   // would re-resolve against the cwd of whoever reads it.
+   return path.resolve(path.dirname(resolved.path));
+};
+
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const resolved = resolvePublisherConfigPath(serverRoot);
    if (!resolved) {

package/src/controller/materialization.controller.spec.ts

@@ -36,6 +36,15 @@ describe("MaterializationController.createMaterialization validation", () => {
       ).toEqual({ forceRefresh: true, sourceNames: ["a", "b"] });
    });
 
+   it("never forwards a client-supplied trigger (SCHEDULER cannot be forged)", async () => {
+      // trigger is service-level-only: the controller must strip it so an API
+      // caller cannot mint a run that reads as scheduler-driven. The service
+      // then defaults it to ON_DEMAND.
+      const parsed = await parse({ forceRefresh: true, trigger: "SCHEDULER" });
+      expect(parsed).toEqual({ forceRefresh: true });
+      expect("trigger" in (parsed as object)).toBe(false);
+   });
+
    it("rejects a non-boolean forceRefresh", async () => {
       const { controller } = build();
       await expect(

package/src/controller/materialization.controller.ts

@@ -20,6 +20,11 @@ export class MaterializationController {
       );
    }
 
+   // Whitelist: only these fields are read off the client body. `trigger` is
+   // deliberately NOT accepted here — it is service-level-only, defaulted to
+   // `ON_DEMAND` and set to `SCHEDULER` solely by the in-process scheduler
+   // (which calls the service directly, not this HTTP path). Do not add
+   // `trigger` to this parser, or an API caller could forge a scheduled run.
    private validateCreateBody(body: Record<string, unknown>): {
       forceRefresh?: boolean;
       sourceNames?: string[];
@@ -201,6 +206,16 @@ export class MaterializationController {
       );
    }
 
+   async listEnvironmentMaterializations(
+      environmentName: string,
+      options?: { limit?: number; offset?: number },
+   ) {
+      return this.materializationService.listEnvironmentMaterializations(
+         environmentName,
+         options,
+      );
+   }
+
    async getMaterialization(
       environmentName: string,
       packageName: string,

package/src/controller/package.controller.ts

@@ -5,6 +5,13 @@ import { EnvironmentStore } from "../service/environment_store";
 
 type ApiPackage = components["schemas"]["Package"];
 
+/**
+ * Which path a reload took. `in-place` recompiles the tree already on disk and
+ * leaves it alone; `reinstalled` re-fetches from the package's install location,
+ * which overwrites on-disk edits.
+ */
+export type PackageReloadMode = "in-place" | "reinstalled";
+
 /**
  * Everything that is strict-at-publish, joined into one 400 message (or
  * undefined when the package is publishable): invalid explores entries plus
@@ -50,45 +57,69 @@ export class PackageController {
       packageName: string,
       reload: boolean,
    ): Promise<ApiPackage> {
+      if (reload) {
+         return (await this.reloadPackage(environmentName, packageName))
+            .metadata;
+      }
+
       const environment = await this.environmentStore.getEnvironment(
          environmentName,
          false,
       );
-
-      if (reload) {
-         // Resolve the package's source location from the currently-cached
-         // metadata WITHOUT triggering a stale-state reload. If a `location`
-         // is set, route the reload through `installPackage` so that
-         // download-then-load happens atomically; otherwise fall back to an
-         // in-place reload of the existing on-disk content.
-         let location: string | undefined;
-         try {
-            const cached = await environment.getPackage(packageName, false);
-            location = cached.getPackageMetadata().location;
-         } catch {
-            // Not previously loaded — nothing to reinstall from.
-         }
-         if (location) {
-            const reinstalled = await environment.installPackage(
-               packageName,
-               (stagingPath) =>
-                  this.downloadInto(
-                     environmentName,
-                     packageName,
-                     location,
-                     stagingPath,
-                  ),
-            );
-            return reinstalled.getPackageMetadata();
-         }
-         const _package = await environment.getPackage(packageName, true);
-         return _package.getPackageMetadata();
-      }
-
       const _package = await environment.getPackage(packageName, false);
       return _package.getPackageMetadata();
    }
 
+   /**
+    * Reload a package and report WHICH path ran. The two are not equivalent to
+    * anyone holding on-disk edits: an in-place reload recompiles the tree that
+    * is already there, while a reinstall re-fetches from the package's install
+    * location and overwrites it. A caller that surfaces the reload to a user
+    * needs to be able to say which happened, so it is returned rather than
+    * inferred. `getPackage` keeps returning bare metadata, so the REST contract
+    * is unchanged.
+    */
+   public async reloadPackage(
+      environmentName: string,
+      packageName: string,
+   ): Promise<{ metadata: ApiPackage; mode: PackageReloadMode }> {
+      const environment = await this.environmentStore.getEnvironment(
+         environmentName,
+         false,
+      );
+
+      // Resolve the package's source location from the currently-cached
+      // metadata WITHOUT triggering a stale-state reload. If a `location`
+      // is set, route the reload through `installPackage` so that
+      // download-then-load happens atomically; otherwise fall back to an
+      // in-place reload of the existing on-disk content.
+      let location: string | undefined;
+      try {
+         const cached = await environment.getPackage(packageName, false);
+         location = cached.getPackageMetadata().location;
+      } catch {
+         // Not previously loaded, so there is nothing to reinstall from.
+      }
+      if (location) {
+         const reinstalled = await environment.installPackage(
+            packageName,
+            (stagingPath) =>
+               this.downloadInto(
+                  environmentName,
+                  packageName,
+                  location,
+                  stagingPath,
+               ),
+         );
+         return {
+            metadata: reinstalled.getPackageMetadata(),
+            mode: "reinstalled",
+         };
+      }
+      const _package = await environment.getPackage(packageName, true);
+      return { metadata: _package.getPackageMetadata(), mode: "in-place" };
+   }
+
    async addPackage(environmentName: string, body: ApiPackage) {
       if (this.environmentStore.publisherConfigIsFrozen) {
          throw new FrozenConfigError();

package/src/default-publisher.config.json

@@ -2,19 +2,19 @@
   "frozenConfig": false,
   "environments": [
     {
-      "name": "malloy-samples",
+      "name": "examples",
       "packages": [
         {
-          "name": "ecommerce",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/ecommerce"
+          "name": "storefront",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/storefront"
         },
         {
-          "name": "imdb",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/imdb"
+          "name": "governed-analytics",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/governed-analytics"
         },
         {
-          "name": "faa",
-          "location": "https://github.com/credibledata/malloy-samples/tree/main/faa"
+          "name": "html-data-app",
+          "location": "https://github.com/malloydata/publisher/tree/main/examples/html-data-app"
         }
       ],
       "connections": []

package/src/materialization_metrics.ts

@@ -78,6 +78,13 @@ const manifestBindCounter = lazyCounter(
    "publisher_materialization_manifest_bind_total",
    "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
 );
+const manifestBindDegradedCounter = lazyCounter(
+   "publisher_materialization_manifest_bind_degraded_total",
+   "Manifest entries bound with an UNQUOTED table path because their connection " +
+      "could not be resolved (serve-side bind) or is absent from the build " +
+      "(build-side seed). A misconfiguration that breaks the source on a " +
+      "case-folding engine (Snowflake); alertable.",
+);
 const sourceBuildDuration = lazyHistogram(
    "publisher_materialization_source_build_duration_ms",
    "Wall-clock duration of building a single persist source.",
@@ -87,6 +94,22 @@ const dropTablesCounter = lazyCounter(
    "publisher_materialization_drop_tables_total",
    "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
 );
+const scheduledFireCounter = lazyCounter(
+   "publisher_materialization_scheduled_fires_total",
+   "Standalone-scheduler attempts to fire a package's materialization.schedule. " +
+      "Label: outcome ('fired'|'conflict'|'error').",
+);
+
+/**
+ * Record a standalone-scheduler fire attempt. `fired` = a SCHEDULER run started;
+ * `conflict` = a materialization was already active (this occurrence coalesces
+ * into it); `error` = the fire failed unexpectedly.
+ */
+export function recordScheduledFire(
+   outcome: "fired" | "conflict" | "error",
+): void {
+   scheduledFireCounter().add(1, { outcome });
+}
 
 /** Record the outcome and duration of a materialization build. */
 export function recordMaterializationRun(
@@ -147,6 +170,16 @@ export function recordManifestBind(outcome: ManifestBindOutcome): void {
    manifestBindCounter().add(1, { outcome });
 }
 
+/**
+ * A single manifest entry bound with an unquoted table path because its
+ * connection could not be resolved for quoting. Per-entry degradation (the rest
+ * of the bind succeeds), distinct from the whole-bind {@link recordManifestBind}
+ * outcome — hence its own counter rather than a label on that one.
+ */
+export function recordManifestBindDegraded(): void {
+   manifestBindDegradedCounter().add(1);
+}
+
 /** Record the wall-clock duration of building one persist source's table. */
 export function recordSourceBuildDuration(durationMs: number): void {
    sourceBuildDuration().record(durationMs);