@malloy-publisher/server 0.0.228 → 0.0.229

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.228",
+   "version": "0.0.229",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -59,6 +59,7 @@
       "body-parser": "^1.20.2",
       "chokidar": "^4.0.3",
       "cors": "^2.8.5",
+      "cron-parser": "^5.6.2",
       "express": "^4.21.0",
       "extract-zip": "^2.0.1",
       "globals": "^15.9.0",
@@ -69,7 +70,8 @@
       "node-cron": "^3.0.3",
       "recursive-readdir": "^2.2.3",
       "simple-git": "^3.28.0",
-      "uuid": "^11.0.3"
+      "uuid": "^11.0.3",
+      "winston": "^3.17.0"
    },
    "devDependencies": {
       "@eslint/compat": "^1.2.7",

package/src/config.spec.ts

@@ -1,7 +1,11 @@
 import { afterEach, beforeEach, describe, expect, it } from "bun:test";
 import fs from "fs";
 import path from "path";
-import { getPublisherConfig, type PublisherConfig } from "./config";
+import {
+   getPublisherConfig,
+   getPublisherConfigDir,
+   type PublisherConfig,
+} from "./config";
 import { PUBLISHER_CONFIG_NAME } from "./constants";
 
 describe("Config Environment Variable Substitution", () => {
@@ -1408,3 +1412,85 @@ describe("getMaxConcurrentQueries", () => {
       expect(() => getMaxConcurrentQueries()).toThrow();
    });
 });
+
+describe("getPublisherConfigDir", () => {
+   const testRoot = path.join(process.cwd(), "test-temp-config-dir");
+   const elsewhere = path.join(testRoot, "elsewhere");
+   const savedConfigPath = process.env.PUBLISHER_CONFIG_PATH;
+
+   beforeEach(() => {
+      fs.mkdirSync(elsewhere, { recursive: true });
+      // Both knobs feed resolvePublisherConfigPath, and every spec file shares
+      // one process, so clear both rather than inherit whatever ran before.
+      delete process.env.PUBLISHER_CONFIG_PATH;
+      delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
+   });
+
+   afterEach(() => {
+      fs.rmSync(testRoot, { recursive: true, force: true });
+      // Clear both, not just the one this describe saves: every spec file shares
+      // one process, so a knob left set here follows whatever runs next.
+      delete process.env.PUBLISHER_USE_BUNDLED_DEFAULT;
+      if (savedConfigPath === undefined) {
+         delete process.env.PUBLISHER_CONFIG_PATH;
+      } else {
+         process.env.PUBLISHER_CONFIG_PATH = savedConfigPath;
+      }
+   });
+
+   it("returns the server root when the config sits there", () => {
+      fs.writeFileSync(
+         path.join(testRoot, PUBLISHER_CONFIG_NAME),
+         JSON.stringify({ environments: [] }),
+      );
+      expect(getPublisherConfigDir(testRoot)).toBe(testRoot);
+   });
+
+   it("follows PUBLISHER_CONFIG_PATH to a config outside the server root", () => {
+      // The --config case: a relative package location must anchor next to
+      // the config that declares it, not at whatever cwd the server booted in.
+      const configPath = path.join(elsewhere, PUBLISHER_CONFIG_NAME);
+      fs.writeFileSync(configPath, JSON.stringify({ environments: [] }));
+      process.env.PUBLISHER_CONFIG_PATH = configPath;
+      expect(getPublisherConfigDir(testRoot)).toBe(elsewhere);
+   });
+
+   it("returns null when no config resolves, so callers fall back to the server root", () => {
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+
+   it("returns an absolute anchor even when --config is relative", () => {
+      // An anchor that is itself relative would re-resolve against the cwd of
+      // whoever reads it, which is the bug this whole change is fixing.
+      const rel = path.relative(
+         process.cwd(),
+         path.join(elsewhere, PUBLISHER_CONFIG_NAME),
+      );
+      fs.writeFileSync(
+         path.join(elsewhere, PUBLISHER_CONFIG_NAME),
+         JSON.stringify({ environments: [] }),
+      );
+      process.env.PUBLISHER_CONFIG_PATH = rel;
+      const anchor = getPublisherConfigDir(testRoot);
+      expect(anchor).not.toBeNull();
+      expect(path.isAbsolute(anchor as string)).toBe(true);
+      expect(anchor).toBe(elsewhere);
+   });
+
+   it("returns null when --config names a directory instead of a file", () => {
+      // existsSync is true for a directory, so without a file check the anchor
+      // would silently become that directory's PARENT, for a config that never
+      // loads (getPublisherConfig catches the read error and returns empty).
+      process.env.PUBLISHER_CONFIG_PATH = elsewhere;
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+
+   it("returns null for the bundled default rather than anchoring inside the install", () => {
+      // Zero-arg `npx @malloy-publisher/server` resolves the config that ships
+      // inside the package. Anchoring a user's relative location under
+      // node_modules would be meaningless, so the caller falls back to the
+      // server root, which is what this did before the anchor moved.
+      process.env.PUBLISHER_USE_BUNDLED_DEFAULT = "true";
+      expect(getPublisherConfigDir(testRoot)).toBeNull();
+   });
+});

package/src/config.ts

@@ -71,7 +71,13 @@ function resolvePublisherConfigPath(serverRoot: string): {
    return null;
 }
 
-type FilesystemPath = `./${string}` | `../${string}` | `/${string}`;
+// Home paths are POSIX form only: `~/` expands, while bare `~`, `~user/…`,
+// and Windows-style `~\` are not local paths and are rejected downstream.
+type FilesystemPath =
+   | `./${string}`
+   | `../${string}`
+   | `/${string}`
+   | `~/${string}`;
 type GcsPath = `gs://${string}`;
 type ApiConnection = components["schemas"]["Connection"];
 export type Theme = components["schemas"]["Theme"];
@@ -256,6 +262,72 @@ export const getMemoryGovernorConfig = (): MemoryGovernorConfig | null => {
    };
 };
 
+/**
+ * Tunables for the standalone {@link MaterializationScheduler}. Sourced from
+ * environment variables at startup; see {@link getMaterializationSchedulerConfig}.
+ *
+ * The scheduler fires a package's `materialization.schedule` cron in **standalone**
+ * deployments (no control plane). It is **disabled by default** — an orchestrated
+ * deployment, whose control plane already drives materialization, never sets the
+ * enable flag, so the scheduler is never constructed there.
+ */
+export interface MaterializationSchedulerConfig {
+   /** Cadence of the due-schedule sweep, in milliseconds. */
+   tickIntervalMs: number;
+   /** Max packages fired per tick — a stampede guard for large deployments. */
+   maxFiresPerTick: number;
+}
+
+const DEFAULT_SCHEDULER_INTERVAL_MS = 60_000;
+const MIN_SCHEDULER_INTERVAL_MS = 1_000;
+const DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK = 10;
+
+/**
+ * Parse standalone-scheduler settings and return a validated config, or `null`
+ * when the feature is disabled. Disabled iff
+ * `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` is unset/false — the default — so
+ * the scheduler never runs in an orchestrated deployment (where the control
+ * plane drives materialization itself).
+ *
+ * **Never set `PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER` on an orchestrated
+ * worker.** It is the primary safety guard: a control-plane-loaded package that
+ * is serving live has `manifestLocation === null`, so the scheduler's
+ * per-package `manifestLocation` skip does not cover it — only this flag being
+ * off keeps the standalone scheduler from double-driving the control plane.
+ *
+ * Throws at startup on malformed input so a typo surfaces loudly rather than
+ * silently disabling scheduling.
+ */
+export const getMaterializationSchedulerConfig =
+   (): MaterializationSchedulerConfig | null => {
+      const enabled =
+         parseBoolEnv("PUBLISHER_LOCAL_MATERIALIZATION_SCHEDULER") ?? false;
+      if (!enabled) {
+         return null;
+      }
+
+      const tickIntervalMs =
+         parseIntEnv("PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS") ??
+         DEFAULT_SCHEDULER_INTERVAL_MS;
+      const maxFiresPerTick =
+         parseIntEnv(
+            "PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK",
+         ) ?? DEFAULT_SCHEDULER_MAX_FIRES_PER_TICK;
+
+      if (tickIntervalMs < MIN_SCHEDULER_INTERVAL_MS) {
+         throw new Error(
+            `PUBLISHER_MATERIALIZATION_SCHEDULER_INTERVAL_MS must be >= ${MIN_SCHEDULER_INTERVAL_MS} (got ${tickIntervalMs})`,
+         );
+      }
+      if (maxFiresPerTick <= 0) {
+         throw new Error(
+            `PUBLISHER_MATERIALIZATION_SCHEDULER_MAX_FIRES_PER_TICK must be a positive integer (got ${maxFiresPerTick})`,
+         );
+      }
+
+      return { tickIntervalMs, maxFiresPerTick };
+   };
+
 /**
  * Resolve the row cap applied to ad-hoc connection SQL queries.
  * Reads `PUBLISHER_MAX_QUERY_ROWS`; falls back to
@@ -404,6 +476,35 @@ function processConfigValue(value: unknown): unknown {
    return value;
 }
 
+/**
+ * Absolute directory that a relative package `location` is resolved against:
+ * the one holding the active config file.
+ *
+ * Null when there is nothing sensible to anchor to, leaving the caller to pick
+ * a base. That covers three cases: no config at all; the bundled default, which
+ * lives inside the installed package, where anchoring a user's relative path
+ * somewhere under node_modules would be meaningless (and which declares only
+ * remote locations of its own); and a `--config` that names a directory rather
+ * than a file, which cannot be read as a config, so its parent is not an anchor
+ * anyone asked for.
+ */
+export const getPublisherConfigDir = (serverRoot: string): string | null => {
+   const resolved = resolvePublisherConfigPath(serverRoot);
+   if (!resolved || resolved.isBundledDefault) {
+      return null;
+   }
+   try {
+      if (!fs.statSync(resolved.path).isFile()) {
+         return null;
+      }
+   } catch {
+      return null;
+   }
+   // Resolve: `--config` may be relative, and an anchor that is itself relative
+   // would re-resolve against the cwd of whoever reads it.
+   return path.resolve(path.dirname(resolved.path));
+};
+
 export const getPublisherConfig = (serverRoot: string): PublisherConfig => {
    const resolved = resolvePublisherConfigPath(serverRoot);
    if (!resolved) {

package/src/controller/materialization.controller.spec.ts

@@ -36,6 +36,15 @@ describe("MaterializationController.createMaterialization validation", () => {
       ).toEqual({ forceRefresh: true, sourceNames: ["a", "b"] });
    });
 
+   it("never forwards a client-supplied trigger (SCHEDULER cannot be forged)", async () => {
+      // trigger is service-level-only: the controller must strip it so an API
+      // caller cannot mint a run that reads as scheduler-driven. The service
+      // then defaults it to ON_DEMAND.
+      const parsed = await parse({ forceRefresh: true, trigger: "SCHEDULER" });
+      expect(parsed).toEqual({ forceRefresh: true });
+      expect("trigger" in (parsed as object)).toBe(false);
+   });
+
    it("rejects a non-boolean forceRefresh", async () => {
       const { controller } = build();
       await expect(

package/src/controller/materialization.controller.ts

@@ -20,6 +20,11 @@ export class MaterializationController {
       );
    }
 
+   // Whitelist: only these fields are read off the client body. `trigger` is
+   // deliberately NOT accepted here — it is service-level-only, defaulted to
+   // `ON_DEMAND` and set to `SCHEDULER` solely by the in-process scheduler
+   // (which calls the service directly, not this HTTP path). Do not add
+   // `trigger` to this parser, or an API caller could forge a scheduled run.
    private validateCreateBody(body: Record<string, unknown>): {
       forceRefresh?: boolean;
       sourceNames?: string[];
@@ -201,6 +206,16 @@ export class MaterializationController {
       );
    }
 
+   async listEnvironmentMaterializations(
+      environmentName: string,
+      options?: { limit?: number; offset?: number },
+   ) {
+      return this.materializationService.listEnvironmentMaterializations(
+         environmentName,
+         options,
+      );
+   }
+
    async getMaterialization(
       environmentName: string,
       packageName: string,

package/src/materialization_metrics.ts

@@ -78,6 +78,13 @@ const manifestBindCounter = lazyCounter(
    "publisher_materialization_manifest_bind_total",
    "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
 );
+const manifestBindDegradedCounter = lazyCounter(
+   "publisher_materialization_manifest_bind_degraded_total",
+   "Manifest entries bound with an UNQUOTED table path because their connection " +
+      "could not be resolved (serve-side bind) or is absent from the build " +
+      "(build-side seed). A misconfiguration that breaks the source on a " +
+      "case-folding engine (Snowflake); alertable.",
+);
 const sourceBuildDuration = lazyHistogram(
    "publisher_materialization_source_build_duration_ms",
    "Wall-clock duration of building a single persist source.",
@@ -87,6 +94,22 @@ const dropTablesCounter = lazyCounter(
    "publisher_materialization_drop_tables_total",
    "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
 );
+const scheduledFireCounter = lazyCounter(
+   "publisher_materialization_scheduled_fires_total",
+   "Standalone-scheduler attempts to fire a package's materialization.schedule. " +
+      "Label: outcome ('fired'|'conflict'|'error').",
+);
+
+/**
+ * Record a standalone-scheduler fire attempt. `fired` = a SCHEDULER run started;
+ * `conflict` = a materialization was already active (this occurrence coalesces
+ * into it); `error` = the fire failed unexpectedly.
+ */
+export function recordScheduledFire(
+   outcome: "fired" | "conflict" | "error",
+): void {
+   scheduledFireCounter().add(1, { outcome });
+}
 
 /** Record the outcome and duration of a materialization build. */
 export function recordMaterializationRun(
@@ -147,6 +170,16 @@ export function recordManifestBind(outcome: ManifestBindOutcome): void {
    manifestBindCounter().add(1, { outcome });
 }
 
+/**
+ * A single manifest entry bound with an unquoted table path because its
+ * connection could not be resolved for quoting. Per-entry degradation (the rest
+ * of the bind succeeds), distinct from the whole-bind {@link recordManifestBind}
+ * outcome — hence its own counter rather than a label on that one.
+ */
+export function recordManifestBindDegraded(): void {
+   manifestBindDegradedCounter().add(1);
+}
+
 /** Record the wall-clock duration of building one persist source's table. */
 export function recordSourceBuildDuration(durationMs: number): void {
    sourceBuildDuration().record(durationMs);

package/src/mcp/skills/build_skills_bundle.ts

@@ -13,7 +13,7 @@
 import * as fs from "fs";
 import * as path from "path";
 
-interface SkillEntry {
+export interface SkillEntry {
    name: string;
    description: string;
    body: string;
@@ -36,6 +36,33 @@ export function parseSkill(md: string, dirName: string): SkillEntry {
    return { name, description, body };
 }
 
+/**
+ * `credible-*` skills are Credible-platform-specific and never ship (see
+ * skills/README.md). They can legitimately sit in skills/ uncommitted: the
+ * directory is a gitignored local install target (`skills/credible-*` in
+ * .gitignore), so the bundle must skip them or a regeneration on a machine
+ * with Credible installed would bake them into the committed bundle, and the
+ * regenerate-and-compare spec would fail on a clean checkout.
+ *
+ * The canonical predicate lives in packages/skills/scripts/exclusions.ts;
+ * duplicated here because the server does not depend on that package.
+ */
+export function isCredible(dirName: string): boolean {
+   return dirName.toLowerCase().startsWith("credible-");
+}
+
+/** Read every skill under a skills directory, in the bundle's own order. */
+export function buildSkills(skillsDir: string): SkillEntry[] {
+   const skills: SkillEntry[] = [];
+   for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+      if (!entry.isDirectory() || isCredible(entry.name)) continue;
+      const skillFile = path.join(skillsDir, entry.name, "SKILL.md");
+      if (!fs.existsSync(skillFile)) continue;
+      skills.push(parseSkill(fs.readFileSync(skillFile, "utf8"), entry.name));
+   }
+   return skills.sort((a, b) => a.name.localeCompare(b.name));
+}
+
 function main(): void {
    const skillsDir = process.argv[2];
    const outFile =
@@ -47,14 +74,7 @@ function main(): void {
       process.exit(1);
    }
 
-   const skills: SkillEntry[] = [];
-   for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
-      if (!entry.isDirectory()) continue;
-      const skillFile = path.join(skillsDir, entry.name, "SKILL.md");
-      if (!fs.existsSync(skillFile)) continue;
-      skills.push(parseSkill(fs.readFileSync(skillFile, "utf8"), entry.name));
-   }
-   skills.sort((a, b) => a.name.localeCompare(b.name));
+   const skills = buildSkills(skillsDir);
 
    fs.writeFileSync(outFile, JSON.stringify({ skills }));
    console.log(`Wrote ${skills.length} skills to ${outFile}`);