@malloy-publisher/server 0.0.229 → 0.0.231

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.229",
+   "version": "0.0.231",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -36,17 +36,17 @@
       "@azure/storage-blob": "^12.26.0",
       "@duckdb/node-api": "1.5.3-r.2",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.422",
-      "@malloydata/db-databricks": "^0.0.422",
-      "@malloydata/db-duckdb": "^0.0.422",
-      "@malloydata/db-mysql": "^0.0.422",
-      "@malloydata/db-postgres": "^0.0.422",
-      "@malloydata/db-publisher": "^0.0.422",
-      "@malloydata/db-snowflake": "^0.0.422",
-      "@malloydata/db-trino": "^0.0.422",
-      "@malloydata/malloy": "^0.0.422",
-      "@malloydata/malloy-sql": "^0.0.422",
-      "@malloydata/render-validator": "^0.0.422",
+      "@malloydata/db-bigquery": "^0.0.423",
+      "@malloydata/db-databricks": "^0.0.423",
+      "@malloydata/db-duckdb": "^0.0.423",
+      "@malloydata/db-mysql": "^0.0.423",
+      "@malloydata/db-postgres": "^0.0.423",
+      "@malloydata/db-publisher": "^0.0.423",
+      "@malloydata/db-snowflake": "^0.0.423",
+      "@malloydata/db-trino": "^0.0.423",
+      "@malloydata/malloy": "^0.0.423",
+      "@malloydata/malloy-sql": "^0.0.423",
+      "@malloydata/render-validator": "^0.0.423",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",

package/scripts/bake-duckdb-extensions.js

@@ -27,8 +27,11 @@ import { DuckDBInstance } from "@duckdb/node-api";
 // sandbox, federated-database attach, and the materialization catalog. Keep this
 // in sync with the install sites in those files.
 //
-// `community: true` mirrors the runtime's `FORCE INSTALL '<name>' FROM community`
-// (bigquery, snowflake); the rest are core extensions installed by name.
+// `community: true` marks the community-repo extensions (bigquery, snowflake);
+// the rest are core extensions installed by name. The bake deliberately uses
+// `FORCE INSTALL ... FROM community` to (re)populate the build cache from the
+// CDN at build time; the *runtime* install (connection.ts) uses a plain,
+// local-first `INSTALL` that no-ops against this baked cache.
 // `registered` is the name the extension reports in duckdb_extensions() when it
 // differs from the INSTALL name (only postgres -> postgres_scanner).
 const EXTENSIONS = [

package/src/config.ts

@@ -440,6 +440,46 @@ export const getMaxConcurrentQueries = (): number => {
    return raw;
 };
 
+/**
+ * DuckDB extension-fetch policy. Governs whether Publisher's explicit extension
+ * INSTALL step (see `installAndLoadExtension` in service/connection.ts) may
+ * reach the DuckDB extension network.
+ */
+export type ExtensionFetchPolicy = "on-demand" | "local-only";
+
+/**
+ * Resolve the DuckDB extension-fetch policy from `EXTENSION_FETCH_POLICY`;
+ * falls back to `on-demand` when unset or empty.
+ *
+ * - `on-demand` (default): preserves prior behaviour. Publisher runs `INSTALL`
+ *   for a missing extension on first use, which fetches it from the DuckDB
+ *   extension network when it is not already present on disk (baked into the
+ *   image). Extensions already baked are used as-is.
+ * - `local-only`: Publisher never runs `INSTALL`, and turns DuckDB's own
+ *   implicit auto-install off on its build/serve sessions
+ *   (`autoinstall_known_extensions=false`), so no code path reaches the
+ *   network. Auto-LOAD stays on, so an extension already present on disk still
+ *   lazy-loads; a genuinely missing extension surfaces as a loud, actionable
+ *   error instead of a silent fetch. For air-gapped / pinned-image deployments.
+ *
+ * Throws on an unrecognised value. Validated at server startup (server.ts calls
+ * this during boot) so a typo in a k8s manifest fails the boot loudly rather
+ * than surfacing on the first query that resolves a DuckDB connection — which,
+ * for a DuckLake-only deployment, could be well after `serving`. Also read on
+ * each connection resolve, so the value is honoured without a restart.
+ */
+export const getExtensionFetchPolicy = (): ExtensionFetchPolicy => {
+   const raw = process.env.EXTENSION_FETCH_POLICY;
+   if (raw === undefined || raw.trim() === "") return "on-demand";
+   const normalised = raw.trim().toLowerCase();
+   if (normalised === "on-demand" || normalised === "local-only") {
+      return normalised;
+   }
+   throw new Error(
+      `Invalid value for EXTENSION_FETCH_POLICY: expected "on-demand" or "local-only", got "${raw}"`,
+   );
+};
+
 function substituteEnvVars(value: string): string {
    const envVarPattern = /\$\{([A-Z_][A-Z0-9_]*)\}/g;
 

package/src/ducklake_version.spec.ts

@@ -0,0 +1,163 @@
+import { describe, expect, it } from "bun:test";
+import { readFileSync } from "fs";
+import { fileURLToPath } from "url";
+import {
+   catalogFormatRangeForEngine,
+   compareCatalogFormat,
+   ENGINE_MAX_FORMAT,
+   isCatalogFormatInRange,
+   isCatalogFormatSupportedByEngine,
+   MIN_CATALOG_FORMAT,
+   parseCatalogFormat,
+   parseEngineMajorMinor,
+} from "./ducklake_version";
+
+/**
+ * Resolve the pinned `@duckdb/node-api` engine version from bun.lock, the same
+ * source `scripts/duckdb-version.js` reads (mirrored here rather than imported
+ * because the server tsconfig has `allowJs: false`). Prefers the version nested
+ * under `@malloydata/db-duckdb` — the engine Malloy actually runs on — and
+ * falls back to the deduped top-level entry.
+ */
+function resolvePinnedDuckDBVersion(): string {
+   const lock = readFileSync(
+      fileURLToPath(new URL("../../../bun.lock", import.meta.url)),
+      "utf8",
+   );
+   const nested = lock.match(
+      /"@malloydata\/db-duckdb\/@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
+   );
+   const plain = lock.match(
+      /"@duckdb\/node-api":\s*\["@duckdb\/node-api@(\d+\.\d+\.\d+[^"]*)"/,
+   );
+   const match = nested ?? plain;
+   if (!match) throw new Error("@duckdb/node-api not found in bun.lock");
+   return match[1];
+}
+
+describe("ducklake_version", () => {
+   describe("parseCatalogFormat", () => {
+      it("parses two-part and pre-release formats", () => {
+         expect(parseCatalogFormat("1.0")).toEqual({ major: 1, minor: 0 });
+         expect(parseCatalogFormat("0.3-dev1")).toEqual({
+            major: 0,
+            minor: 3,
+            prerelease: "dev1",
+         });
+      });
+
+      it("rejects malformed input without throwing", () => {
+         // Three-part strings are malformed for a catalog format, not a newer
+         // version -- they must not parse as a bigger number.
+         expect(parseCatalogFormat("1.0.0")).toBeNull();
+         expect(parseCatalogFormat("1.1.0")).toBeNull();
+         expect(parseCatalogFormat("")).toBeNull();
+         expect(parseCatalogFormat("not a version")).toBeNull();
+         expect(parseCatalogFormat("0.3 ")).toBeNull();
+      });
+   });
+
+   describe("compareCatalogFormat", () => {
+      const p = (s: string) => parseCatalogFormat(s)!;
+      it("orders by major, then minor", () => {
+         expect(compareCatalogFormat(p("0.3"), p("1.0"))).toBe(-1);
+         expect(compareCatalogFormat(p("1.1"), p("1.0"))).toBe(1);
+         expect(compareCatalogFormat(p("1.0"), p("1.0"))).toBe(0);
+      });
+      it("sorts a release above its own pre-releases", () => {
+         expect(compareCatalogFormat(p("1.0"), p("1.0-dev1"))).toBe(1);
+         expect(compareCatalogFormat(p("0.3-dev1"), p("0.3"))).toBe(-1);
+      });
+   });
+
+   describe("parseEngineMajorMinor", () => {
+      it("extracts the minor line from assorted engine spec forms", () => {
+         expect(parseEngineMajorMinor("1.5.3")).toEqual({ major: 1, minor: 5 });
+         expect(parseEngineMajorMinor("1.5.3-r.2")).toEqual({
+            major: 1,
+            minor: 5,
+         });
+         expect(parseEngineMajorMinor("v1.5.3")).toEqual({
+            major: 1,
+            minor: 5,
+         });
+      });
+      it("returns null when no version is present", () => {
+         expect(parseEngineMajorMinor("nope")).toBeNull();
+      });
+   });
+
+   describe("catalogFormatRangeForEngine", () => {
+      it("derives [1.0, maxFormat] for a covered engine (all patch levels)", () => {
+         expect(catalogFormatRangeForEngine("1.5.3")).toEqual({
+            min: "1.0",
+            max: "1.0",
+            engineVersion: "1.5",
+         });
+         // Any patch of a known minor line resolves to the same range.
+         expect(catalogFormatRangeForEngine("1.5.9")).toEqual({
+            min: "1.0",
+            max: "1.0",
+            engineVersion: "1.5",
+         });
+      });
+      it("returns null (drift signal) for an engine with no matrix row", () => {
+         expect(catalogFormatRangeForEngine("1.6.0")).toBeNull();
+         expect(catalogFormatRangeForEngine("2.0.0")).toBeNull();
+         expect(catalogFormatRangeForEngine("garbage")).toBeNull();
+      });
+   });
+
+   // These reproduce the accept/reject set the prior enumerated-list work
+   // asserted, now expressed as a RANGE derived from the 1.5.x engine.
+   describe("isCatalogFormatSupportedByEngine (engine 1.5.x)", () => {
+      const engine = "1.5.3";
+      it("accepts the 1.0 catalog format produced by the 1.5.x engine", () => {
+         expect(isCatalogFormatSupportedByEngine("1.0", engine)).toBe(true);
+      });
+      it("rejects the 0.3-line formats (attach only via migration)", () => {
+         for (const v of ["0.1", "0.2", "0.3-dev1", "0.3"]) {
+            expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
+         }
+      });
+      it("rejects formats newer than the supported max", () => {
+         for (const v of ["1.1", "2.0", "0.4", "0.3-dev2"]) {
+            expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
+         }
+      });
+      it("rejects malformed / three-part input without throwing", () => {
+         for (const v of ["1.0.0", "1.1.0", "", "not a version", "0.3 "]) {
+            expect(isCatalogFormatSupportedByEngine(v, engine)).toBe(false);
+         }
+      });
+      it("rejects everything when the engine is unknown (no matrix row)", () => {
+         expect(isCatalogFormatSupportedByEngine("1.0", "1.6.0")).toBe(false);
+      });
+   });
+
+   describe("isCatalogFormatInRange", () => {
+      const range = catalogFormatRangeForEngine("1.5.3")!;
+      it("is inclusive at both ends", () => {
+         expect(isCatalogFormatInRange("1.0", range)).toBe(true);
+      });
+      it("rejects below min and above max", () => {
+         expect(isCatalogFormatInRange("0.9", range)).toBe(false);
+         expect(isCatalogFormatInRange("1.1", range)).toBe(false);
+      });
+   });
+
+   // The version-contract: the ENGINE_MAX_FORMAT matrix MUST cover the DuckDB
+   // engine this repo is pinned to. This is the same drift guard the CI script
+   // enforces (scripts/validate-ducklake-catalog-range.ts); asserting it here
+   // fails `bun test` the moment a Malloy bump moves the engine past the matrix.
+   describe("pinned-engine drift contract", () => {
+      it("has a matrix row covering the pinned @duckdb/node-api engine", () => {
+         const engineVersion = resolvePinnedDuckDBVersion();
+         const range = catalogFormatRangeForEngine(engineVersion);
+         expect(range).not.toBeNull();
+         expect(range!.min).toBe(MIN_CATALOG_FORMAT);
+         // The matrix must not be empty and its rows must be well-formed.
+         expect(ENGINE_MAX_FORMAT.length).toBeGreaterThan(0);
+      });
+   });
+});

package/src/ducklake_version.ts

@@ -0,0 +1,153 @@
+// DuckLake catalog-format version contract.
+//
+// A DuckLake catalog records the on-disk format version it was written at. The
+// DuckLake extension bundled with a given DuckDB engine attaches only a bounded
+// RANGE of catalog formats without migration; a catalog outside that range
+// fails deep inside DuckDB with an opaque error. This module derives the
+// supported range purely from the pinned DuckDB engine version, so it never has
+// to enumerate an allow-list of "known good" catalog versions (which drifts
+// silently on an engine bump and is easy to forget to update). Both the runtime
+// attach preflight (`service/connection.ts`) and the CI version-contract check
+// (`scripts/validate-ducklake-catalog-range.ts`) consume this one source.
+//
+// Lower bound: fixed at "1.0". The 1.x DuckLake line does not attach the older
+// 0.x formats without AUTOMATIC_MIGRATION, and the Publisher attach paths never
+// pass that flag, so 1.0 is the floor. Upper bound: the maximum format the
+// engine's bundled extension writes/attaches, looked up in the migration matrix
+// below and keyed by engine (major, minor).
+
+/** A parsed DuckLake catalog format version, e.g. `1.0` or `0.3-dev1`. */
+export interface CatalogFormat {
+   major: number;
+   minor: number;
+   /** e.g. `dev1` in `0.3-dev1`; a pre-release sorts BELOW the same release. */
+   prerelease?: string;
+}
+
+/** The inclusive `[min, max]` catalog-format range for a given engine. */
+export interface CatalogFormatRange {
+   min: string;
+   max: string;
+   /** The engine minor line the range was derived from (for error messages). */
+   engineVersion: string;
+}
+
+/** Fixed lower bound of the supported range (see file header). */
+export const MIN_CATALOG_FORMAT = "1.0";
+
+/**
+ * Migration matrix: DuckDB engine minor line -> the maximum DuckLake catalog
+ * format its bundled DuckLake extension attaches WITHOUT migration. This is NOT
+ * an allow-list of catalog versions -- it is the single fact, per engine minor
+ * line, from which the `[MIN_CATALOG_FORMAT, maxFormat]` range is derived.
+ *
+ * Verified against the baked extension: DuckDB 1.5.x's DuckLake extension writes
+ * and attaches the "1.0" catalog format (the older 0.x formats require
+ * AUTOMATIC_MIGRATION, which Publisher never passes). When a Malloy bump moves
+ * `@duckdb/node-api` to a new minor line, add its row here after verifying the
+ * extension's max attach-without-migration format -- the CI version-contract
+ * check FAILS THE BUILD until a row covers the pinned engine, so this never
+ * drifts silently.
+ */
+export const ENGINE_MAX_FORMAT: ReadonlyArray<{
+   engine: { major: number; minor: number };
+   maxFormat: string;
+}> = [{ engine: { major: 1, minor: 5 }, maxFormat: "1.0" }];
+
+const CATALOG_FORMAT_RE = /^(\d+)\.(\d+)(?:-([0-9A-Za-z.]+))?$/;
+
+/**
+ * Parse a catalog format string into its components, or `null` if it is not a
+ * valid `<major>.<minor>[-<prerelease>]` format. A three-part version like
+ * `1.0.0` is intentionally rejected: catalog formats are two-part, and a
+ * three-part string is a malformed input, not a newer format.
+ */
+export function parseCatalogFormat(raw: string): CatalogFormat | null {
+   if (typeof raw !== "string") return null;
+   const m = CATALOG_FORMAT_RE.exec(raw);
+   if (!m) return null;
+   const format: CatalogFormat = { major: Number(m[1]), minor: Number(m[2]) };
+   if (m[3] !== undefined) format.prerelease = m[3];
+   return format;
+}
+
+/**
+ * Compare two parsed catalog formats: -1 if `a < b`, 0 if equal, 1 if `a > b`.
+ * A release sorts ABOVE its own pre-releases (`1.0` > `1.0-dev1`), matching
+ * semver ordering.
+ */
+export function compareCatalogFormat(
+   a: CatalogFormat,
+   b: CatalogFormat,
+): number {
+   if (a.major !== b.major) return a.major < b.major ? -1 : 1;
+   if (a.minor !== b.minor) return a.minor < b.minor ? -1 : 1;
+   if (a.prerelease === b.prerelease) return 0;
+   if (a.prerelease === undefined) return 1; // release > pre-release
+   if (b.prerelease === undefined) return -1;
+   return a.prerelease < b.prerelease ? -1 : 1;
+}
+
+/**
+ * Extract the `{ major, minor }` line from a DuckDB engine version string
+ * (`1.5.3`, `1.5.3-r.2`, `v1.5.3` all parse to `{1, 5}`), or `null` if none is
+ * present.
+ */
+export function parseEngineMajorMinor(
+   raw: string,
+): { major: number; minor: number } | null {
+   const m = /(\d+)\.(\d+)\.(\d+)/.exec(String(raw));
+   if (!m) return null;
+   return { major: Number(m[1]), minor: Number(m[2]) };
+}
+
+/**
+ * The supported catalog-format range for a DuckDB engine version, or `null`
+ * when the engine's minor line has no row in {@link ENGINE_MAX_FORMAT}. A
+ * `null` return is the DRIFT signal: the CI version-contract check fails the
+ * build on it, and the runtime preflight treats it as "unknown, skip" (the
+ * preflight is non-load-bearing). The match is by `(major, minor)` so any patch
+ * release of a known minor line resolves.
+ */
+export function catalogFormatRangeForEngine(
+   engineVersion: string,
+): CatalogFormatRange | null {
+   const e = parseEngineMajorMinor(engineVersion);
+   if (!e) return null;
+   const entry = ENGINE_MAX_FORMAT.find(
+      (r) => r.engine.major === e.major && r.engine.minor === e.minor,
+   );
+   if (!entry) return null;
+   return {
+      min: MIN_CATALOG_FORMAT,
+      max: entry.maxFormat,
+      engineVersion: `${e.major}.${e.minor}`,
+   };
+}
+
+/** Whether `format` falls within the inclusive range (invalid input -> false). */
+export function isCatalogFormatInRange(
+   format: string,
+   range: CatalogFormatRange,
+): boolean {
+   const f = parseCatalogFormat(format);
+   const min = parseCatalogFormat(range.min);
+   const max = parseCatalogFormat(range.max);
+   if (!f || !min || !max) return false;
+   return (
+      compareCatalogFormat(f, min) >= 0 && compareCatalogFormat(f, max) <= 0
+   );
+}
+
+/**
+ * Whether a catalog `format` is supported by the DuckLake extension bundled
+ * with `engineVersion`. Returns `false` for an unknown engine (no matrix row)
+ * or a malformed format.
+ */
+export function isCatalogFormatSupportedByEngine(
+   format: string,
+   engineVersion: string,
+): boolean {
+   const range = catalogFormatRangeForEngine(engineVersion);
+   return range ? isCatalogFormatInRange(format, range) : false;
+}

package/src/errors.ts

@@ -22,6 +22,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(404, error.message);
    } else if (error instanceof ConnectionAuthError) {
       return httpError(422, error.message);
+   } else if (error instanceof UnsupportedCatalogFormatError) {
+      return httpError(422, error.message);
    } else if (error instanceof ModelCompilationError) {
       return httpError(424, error.message);
    } else if (error instanceof ConnectionError) {
@@ -101,6 +103,16 @@ export class ConnectionAuthError extends Error {
    }
 }
 
+// A catalog was reached and authenticated fine, but its on-disk format is
+// outside the range the pinned engine's extension can attach (see
+// ducklake_version.ts). Distinct from ConnectionAuthError so the 422 doesn't
+// read as a credentials problem. Maps to HTTP 422.
+export class UnsupportedCatalogFormatError extends Error {
+   constructor(message: string) {
+      super(message);
+   }
+}
+
 export class ModelCompilationError extends Error {
    // Accepts a MalloyError or any message-bearing object, so callers that add
    // context around a compile failure (e.g. naming the source whose authorize

package/src/malloy_pin_prereqs.spec.ts

@@ -0,0 +1,25 @@
+import { describe, expect, it } from "bun:test";
+import type { VirtualMap } from "@malloydata/malloy";
+
+// Prerequisite guard for the DuckLake materialization tier's build path.
+//
+// The tier resolves persist/virtual sources by handing the Malloy compile a
+// `virtualMap` (connectionName → virtualName → tablePath). This asserts the
+// pinned `@malloydata/malloy` still exports that public `VirtualMap` type, so a
+// pin bump that drops or renames it fails CI typecheck here — turning a
+// point-in-time manual confirm into a durable guard. Confirmed present in the
+// shipped pin (^0.0.422): `VirtualMap = Map<string, Map<string, string>>` is
+// exported from the package root and rides `PrepareResultOptions.virtualMap`.
+describe("malloy pin prerequisites (DuckLake tier)", () => {
+   it("exposes the VirtualMap substitution type in the pinned malloy", () => {
+      // Compile-time guard: this type reference fails typecheck if `VirtualMap`
+      // is removed/renamed in a future pin. The runtime assertions below double
+      // as a shape check on the nested Map<string, Map<string, string>>.
+      const virtualMap: VirtualMap = new Map([
+         ["main_pg", new Map([["orders", "catalog.public.orders"]])],
+      ]);
+      expect(virtualMap.get("main_pg")?.get("orders")).toBe(
+         "catalog.public.orders",
+      );
+   });
+});