@malloy-publisher/server 0.0.228 → 0.0.230

package/src/ducklake_version.ts

@@ -0,0 +1,153 @@
+// DuckLake catalog-format version contract.
+//
+// A DuckLake catalog records the on-disk format version it was written at. The
+// DuckLake extension bundled with a given DuckDB engine attaches only a bounded
+// RANGE of catalog formats without migration; a catalog outside that range
+// fails deep inside DuckDB with an opaque error. This module derives the
+// supported range purely from the pinned DuckDB engine version, so it never has
+// to enumerate an allow-list of "known good" catalog versions (which drifts
+// silently on an engine bump and is easy to forget to update). Both the runtime
+// attach preflight (`service/connection.ts`) and the CI version-contract check
+// (`scripts/validate-ducklake-catalog-range.ts`) consume this one source.
+//
+// Lower bound: fixed at "1.0". The 1.x DuckLake line does not attach the older
+// 0.x formats without AUTOMATIC_MIGRATION, and the Publisher attach paths never
+// pass that flag, so 1.0 is the floor. Upper bound: the maximum format the
+// engine's bundled extension writes/attaches, looked up in the migration matrix
+// below and keyed by engine (major, minor).
+
+/** A parsed DuckLake catalog format version, e.g. `1.0` or `0.3-dev1`. */
+export interface CatalogFormat {
+   major: number;
+   minor: number;
+   /** e.g. `dev1` in `0.3-dev1`; a pre-release sorts BELOW the same release. */
+   prerelease?: string;
+}
+
+/** The inclusive `[min, max]` catalog-format range for a given engine. */
+export interface CatalogFormatRange {
+   min: string;
+   max: string;
+   /** The engine minor line the range was derived from (for error messages). */
+   engineVersion: string;
+}
+
+/** Fixed lower bound of the supported range (see file header). */
+export const MIN_CATALOG_FORMAT = "1.0";
+
+/**
+ * Migration matrix: DuckDB engine minor line -> the maximum DuckLake catalog
+ * format its bundled DuckLake extension attaches WITHOUT migration. This is NOT
+ * an allow-list of catalog versions -- it is the single fact, per engine minor
+ * line, from which the `[MIN_CATALOG_FORMAT, maxFormat]` range is derived.
+ *
+ * Verified against the baked extension: DuckDB 1.5.x's DuckLake extension writes
+ * and attaches the "1.0" catalog format (the older 0.x formats require
+ * AUTOMATIC_MIGRATION, which Publisher never passes). When a Malloy bump moves
+ * `@duckdb/node-api` to a new minor line, add its row here after verifying the
+ * extension's max attach-without-migration format -- the CI version-contract
+ * check FAILS THE BUILD until a row covers the pinned engine, so this never
+ * drifts silently.
+ */
+export const ENGINE_MAX_FORMAT: ReadonlyArray<{
+   engine: { major: number; minor: number };
+   maxFormat: string;
+}> = [{ engine: { major: 1, minor: 5 }, maxFormat: "1.0" }];
+
+const CATALOG_FORMAT_RE = /^(\d+)\.(\d+)(?:-([0-9A-Za-z.]+))?$/;
+
+/**
+ * Parse a catalog format string into its components, or `null` if it is not a
+ * valid `<major>.<minor>[-<prerelease>]` format. A three-part version like
+ * `1.0.0` is intentionally rejected: catalog formats are two-part, and a
+ * three-part string is a malformed input, not a newer format.
+ */
+export function parseCatalogFormat(raw: string): CatalogFormat | null {
+   if (typeof raw !== "string") return null;
+   const m = CATALOG_FORMAT_RE.exec(raw);
+   if (!m) return null;
+   const format: CatalogFormat = { major: Number(m[1]), minor: Number(m[2]) };
+   if (m[3] !== undefined) format.prerelease = m[3];
+   return format;
+}
+
+/**
+ * Compare two parsed catalog formats: -1 if `a < b`, 0 if equal, 1 if `a > b`.
+ * A release sorts ABOVE its own pre-releases (`1.0` > `1.0-dev1`), matching
+ * semver ordering.
+ */
+export function compareCatalogFormat(
+   a: CatalogFormat,
+   b: CatalogFormat,
+): number {
+   if (a.major !== b.major) return a.major < b.major ? -1 : 1;
+   if (a.minor !== b.minor) return a.minor < b.minor ? -1 : 1;
+   if (a.prerelease === b.prerelease) return 0;
+   if (a.prerelease === undefined) return 1; // release > pre-release
+   if (b.prerelease === undefined) return -1;
+   return a.prerelease < b.prerelease ? -1 : 1;
+}
+
+/**
+ * Extract the `{ major, minor }` line from a DuckDB engine version string
+ * (`1.5.3`, `1.5.3-r.2`, `v1.5.3` all parse to `{1, 5}`), or `null` if none is
+ * present.
+ */
+export function parseEngineMajorMinor(
+   raw: string,
+): { major: number; minor: number } | null {
+   const m = /(\d+)\.(\d+)\.(\d+)/.exec(String(raw));
+   if (!m) return null;
+   return { major: Number(m[1]), minor: Number(m[2]) };
+}
+
+/**
+ * The supported catalog-format range for a DuckDB engine version, or `null`
+ * when the engine's minor line has no row in {@link ENGINE_MAX_FORMAT}. A
+ * `null` return is the DRIFT signal: the CI version-contract check fails the
+ * build on it, and the runtime preflight treats it as "unknown, skip" (the
+ * preflight is non-load-bearing). The match is by `(major, minor)` so any patch
+ * release of a known minor line resolves.
+ */
+export function catalogFormatRangeForEngine(
+   engineVersion: string,
+): CatalogFormatRange | null {
+   const e = parseEngineMajorMinor(engineVersion);
+   if (!e) return null;
+   const entry = ENGINE_MAX_FORMAT.find(
+      (r) => r.engine.major === e.major && r.engine.minor === e.minor,
+   );
+   if (!entry) return null;
+   return {
+      min: MIN_CATALOG_FORMAT,
+      max: entry.maxFormat,
+      engineVersion: `${e.major}.${e.minor}`,
+   };
+}
+
+/** Whether `format` falls within the inclusive range (invalid input -> false). */
+export function isCatalogFormatInRange(
+   format: string,
+   range: CatalogFormatRange,
+): boolean {
+   const f = parseCatalogFormat(format);
+   const min = parseCatalogFormat(range.min);
+   const max = parseCatalogFormat(range.max);
+   if (!f || !min || !max) return false;
+   return (
+      compareCatalogFormat(f, min) >= 0 && compareCatalogFormat(f, max) <= 0
+   );
+}
+
+/**
+ * Whether a catalog `format` is supported by the DuckLake extension bundled
+ * with `engineVersion`. Returns `false` for an unknown engine (no matrix row)
+ * or a malformed format.
+ */
+export function isCatalogFormatSupportedByEngine(
+   format: string,
+   engineVersion: string,
+): boolean {
+   const range = catalogFormatRangeForEngine(engineVersion);
+   return range ? isCatalogFormatInRange(format, range) : false;
+}

package/src/errors.ts

@@ -22,6 +22,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(404, error.message);
    } else if (error instanceof ConnectionAuthError) {
       return httpError(422, error.message);
+   } else if (error instanceof UnsupportedCatalogFormatError) {
+      return httpError(422, error.message);
    } else if (error instanceof ModelCompilationError) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
@@ -101,6 +103,16 @@ export class ConnectionAuthError extends Error {
    }
 }
 
+// A catalog was reached and authenticated fine, but its on-disk format is
+// outside the range the pinned engine's extension can attach (see
+// ducklake_version.ts). Distinct from ConnectionAuthError so the 422 doesn't
+// read as a credentials problem. Maps to HTTP 422.
+export class UnsupportedCatalogFormatError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
 export class ModelCompilationError extends Error {
    // Accepts a MalloyError or any message-bearing object, so callers that add
    // context around a compile failure (e.g. naming the source whose authorize

package/src/malloy_pin_prereqs.spec.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "bun:test";
+import type { VirtualMap } from "@malloydata/malloy";
+
+// Prerequisite guard for the DuckLake materialization tier's build path.
+//
+// The tier resolves persist/virtual sources by handing the Malloy compile a
+// `virtualMap` (connectionName → virtualName → tablePath). This asserts the
+// pinned `@malloydata/malloy` still exports that public `VirtualMap` type, so a
+// pin bump that drops or renames it fails CI typecheck here — turning a
+// point-in-time manual confirm into a durable guard. Confirmed present in the
+// shipped pin (^0.0.422): `VirtualMap = Map<string, Map<string, string>>` is
+// exported from the package root and rides `PrepareResultOptions.virtualMap`.
+describe("malloy pin prerequisites (DuckLake tier)", () => {
+   it("exposes the VirtualMap substitution type in the pinned malloy", () => {
+      // Compile-time guard: this type reference fails typecheck if `VirtualMap`
+      // is removed/renamed in a future pin. The runtime assertions below double
+      // as a shape check on the nested Map<string, Map<string, string>>.
+      const virtualMap: VirtualMap = new Map([
+         ["main_pg", new Map([["orders", "catalog.public.orders"]])],
+      ]);
+      expect(virtualMap.get("main_pg")?.get("orders")).toBe(
+         "catalog.public.orders",
+      );
+   });
+});

package/src/materialization_metrics.ts

@@ -78,6 +78,13 @@ const manifestBindCounter = lazyCounter(
    "publisher_materialization_manifest_bind_total",
    "Manifest bind attempts. Label: outcome ('success'|'failure'|'timeout').",
 );
+const manifestBindDegradedCounter = lazyCounter(
+   "publisher_materialization_manifest_bind_degraded_total",
+   "Manifest entries bound with an UNQUOTED table path because their connection " +
+      "could not be resolved (serve-side bind) or is absent from the build " +
+      "(build-side seed). A misconfiguration that breaks the source on a " +
+      "case-folding engine (Snowflake); alertable.",
+);
 const sourceBuildDuration = lazyHistogram(
    "publisher_materialization_source_build_duration_ms",
    "Wall-clock duration of building a single persist source.",
@@ -87,6 +94,22 @@ const dropTablesCounter = lazyCounter(
    "publisher_materialization_drop_tables_total",
    "Physical tables dropped on delete. Label: outcome ('success'|'failure').",
 );
+const scheduledFireCounter = lazyCounter(
+   "publisher_materialization_scheduled_fires_total",
+   "Standalone-scheduler attempts to fire a package's materialization.schedule. " +
+      "Label: outcome ('fired'|'conflict'|'error').",
+);
+
+/**
+ * Record a standalone-scheduler fire attempt. `fired` = a SCHEDULER run started;
+ * `conflict` = a materialization was already active (this occurrence coalesces
+ * into it); `error` = the fire failed unexpectedly.
+ */
+export function recordScheduledFire(
+   outcome: "fired" | "conflict" | "error",
+): void {
+   scheduledFireCounter().add(1, { outcome });
+}
 
 /** Record the outcome and duration of a materialization build. */
 export function recordMaterializationRun(
@@ -147,6 +170,16 @@ export function recordManifestBind(outcome: ManifestBindOutcome): void {
    manifestBindCounter().add(1, { outcome });
 }
 
+/**
+ * A single manifest entry bound with an unquoted table path because its
+ * connection could not be resolved for quoting. Per-entry degradation (the rest
+ * of the bind succeeds), distinct from the whole-bind {@link recordManifestBind}
+ * outcome — hence its own counter rather than a label on that one.
+ */
+export function recordManifestBindDegraded(): void {
+   manifestBindDegradedCounter().add(1);
+}
+
 /** Record the wall-clock duration of building one persist source's table. */
 export function recordSourceBuildDuration(durationMs: number): void {
    sourceBuildDuration().record(durationMs);

package/src/mcp/skills/build_skills_bundle.ts

@@ -13,7 +13,7 @@
 import * as fs from "fs";
 import * as path from "path";
 
-interface SkillEntry {
+export interface SkillEntry {
    name: string;
    description: string;
    body: string;
@@ -36,6 +36,33 @@ export function parseSkill(md: string, dirName: string): SkillEntry {
    return { name, description, body };
 }
 
+/**
+ * `credible-*` skills are Credible-platform-specific and never ship (see
+ * skills/README.md). They can legitimately sit in skills/ uncommitted: the
+ * directory is a gitignored local install target (`skills/credible-*` in
+ * .gitignore), so the bundle must skip them or a regeneration on a machine
+ * with Credible installed would bake them into the committed bundle, and the
+ * regenerate-and-compare spec would fail on a clean checkout.
+ *
+ * The canonical predicate lives in packages/skills/scripts/exclusions.ts;
+ * duplicated here because the server does not depend on that package.
+ */
+export function isCredible(dirName: string): boolean {
+   return dirName.toLowerCase().startsWith("credible-");
+}
+
+/** Read every skill under a skills directory, in the bundle's own order. */
+export function buildSkills(skillsDir: string): SkillEntry[] {
+   const skills: SkillEntry[] = [];
+   for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
+      if (!entry.isDirectory() || isCredible(entry.name)) continue;
+      const skillFile = path.join(skillsDir, entry.name, "SKILL.md");
+      if (!fs.existsSync(skillFile)) continue;
+      skills.push(parseSkill(fs.readFileSync(skillFile, "utf8"), entry.name));
+   }
+   return skills.sort((a, b) => a.name.localeCompare(b.name));
+}
+
 function main(): void {
    const skillsDir = process.argv[2];
    const outFile =
@@ -47,14 +74,7 @@ function main(): void {
       process.exit(1);
    }
 
-   const skills: SkillEntry[] = [];
-   for (const entry of fs.readdirSync(skillsDir, { withFileTypes: true })) {
-      if (!entry.isDirectory()) continue;
-      const skillFile = path.join(skillsDir, entry.name, "SKILL.md");
-      if (!fs.existsSync(skillFile)) continue;
-      skills.push(parseSkill(fs.readFileSync(skillFile, "utf8"), entry.name));
-   }
-   skills.sort((a, b) => a.name.localeCompare(b.name));
+   const skills = buildSkills(skillsDir);
 
    fs.writeFileSync(outFile, JSON.stringify({ skills }));
    console.log(`Wrote ${skills.length} skills to ${outFile}`);