@malloy-publisher/server 0.0.202 → 0.0.204

package/package.json

@@ -1,7 +1,7 @@
 {
    "name": "@malloy-publisher/server",
    "description": "Malloy Publisher Server",
-   "version": "0.0.202",
+   "version": "0.0.204",
    "main": "dist/server.mjs",
    "bin": {
       "malloy-publisher": "dist/server.mjs"
@@ -34,16 +34,16 @@
       "@azure/identity": "^4.13.0",
       "@azure/storage-blob": "^12.26.0",
       "@google-cloud/storage": "^7.16.0",
-      "@malloydata/db-bigquery": "^0.0.394",
-      "@malloydata/db-databricks": "^0.0.394",
-      "@malloydata/db-duckdb": "^0.0.394",
-      "@malloydata/db-mysql": "^0.0.394",
-      "@malloydata/db-postgres": "^0.0.394",
-      "@malloydata/db-snowflake": "^0.0.394",
-      "@malloydata/db-trino": "^0.0.394",
-      "@malloydata/malloy": "^0.0.394",
-      "@malloydata/malloy-sql": "^0.0.394",
-      "@malloydata/render-validator": "^0.0.394",
+      "@malloydata/db-bigquery": "^0.0.405",
+      "@malloydata/db-databricks": "^0.0.405",
+      "@malloydata/db-duckdb": "^0.0.405",
+      "@malloydata/db-mysql": "^0.0.405",
+      "@malloydata/db-postgres": "^0.0.405",
+      "@malloydata/db-snowflake": "^0.0.405",
+      "@malloydata/db-trino": "^0.0.405",
+      "@malloydata/malloy": "^0.0.405",
+      "@malloydata/malloy-sql": "^0.0.405",
+      "@malloydata/render-validator": "^0.0.405",
       "@modelcontextprotocol/sdk": "^1.13.2",
       "@opentelemetry/api": "^1.9.0",
       "@opentelemetry/auto-instrumentations-node": "^0.57.0",

package/src/ducklake_version.spec.ts

@@ -0,0 +1,43 @@
+import { describe, expect, it } from "bun:test";
+import {
+   isCatalogVersionSupported,
+   SUPPORTED_CATALOG_VERSIONS,
+} from "./ducklake_version";
+
+describe("ducklake_version", () => {
+   describe("SUPPORTED_CATALOG_VERSIONS", () => {
+      it("matches the 0.3-line DuckLake extension's internal list", () => {
+         expect([...SUPPORTED_CATALOG_VERSIONS]).toEqual([
+            "0.1",
+            "0.2",
+            "0.3-dev1",
+            "0.3",
+         ]);
+      });
+   });
+
+   describe("isCatalogVersionSupported", () => {
+      it("accepts each documented 0.3-line version", () => {
+         for (const v of SUPPORTED_CATALOG_VERSIONS) {
+            expect(isCatalogVersionSupported(v)).toBe(true);
+         }
+      });
+
+      it("rejects 1.0-line versions", () => {
+         expect(isCatalogVersionSupported("1.0")).toBe(false);
+         expect(isCatalogVersionSupported("1.0.0")).toBe(false);
+         expect(isCatalogVersionSupported("1.1.0")).toBe(false);
+      });
+
+      it("rejects newer pre-1.0 versions not in the supported list", () => {
+         expect(isCatalogVersionSupported("0.4")).toBe(false);
+         expect(isCatalogVersionSupported("0.3-dev2")).toBe(false);
+      });
+
+      it("rejects empty and malformed input without throwing", () => {
+         expect(isCatalogVersionSupported("")).toBe(false);
+         expect(isCatalogVersionSupported("not a version")).toBe(false);
+         expect(isCatalogVersionSupported("0.3 ")).toBe(false);
+      });
+   });
+});

package/src/ducklake_version.ts

@@ -0,0 +1,26 @@
+// DuckLake catalog format compatibility check. Lives at `src/` root so
+// both `service/` (user-facing connections) and `storage/` (materialization
+// catalog) can pre-flight a catalog's recorded version without bringing in
+// the other layer's helpers. See CLAUDE.md's "Two parallel DuckLake/PG
+// attach paths" note.
+//
+// The Publisher Docker image bakes a specific DuckLake extension version at
+// build time (see Dockerfile). If a catalog's on-disk format version is
+// newer than what the baked extension supports, ATTACH fails deep inside
+// DuckDB with a generic 500. The preflight catches that case and surfaces
+// it as a non-retryable 422 instead.
+
+// Versions the currently-baked DuckLake extension (0.3 line, paired with
+// duckdb@1.4.4) can read. When the Publisher upgrades to the DuckLake 1.0
+// line (separate PR, requires duckdb@1.5+), extend this list to include
+// "1.0" and later format versions.
+export const SUPPORTED_CATALOG_VERSIONS: readonly string[] = [
+   "0.1",
+   "0.2",
+   "0.3-dev1",
+   "0.3",
+];
+
+export function isCatalogVersionSupported(version: string): boolean {
+   return SUPPORTED_CATALOG_VERSIONS.includes(version);
+}

package/src/errors.ts

@@ -6,6 +6,8 @@ export function internalErrorToHttpError(error: Error) {
       return httpError(400, error.message);
    } else if (error instanceof FrozenConfigError) {
       return httpError(403, error.message);
+   } else if (error instanceof AccessDeniedError) {
+      return httpError(403, error.message);
    } else if (error instanceof EnvironmentNotFoundError) {
       return httpError(404, error.message);
    } else if (error instanceof PackageNotFoundError) {
@@ -98,7 +100,10 @@ export class ConnectionAuthError extends Error {
 }
 
 export class ModelCompilationError extends Error {
-   constructor(error: MalloyError) {
+   // Accepts a MalloyError or any message-bearing object, so callers that add
+   // context around a compile failure (e.g. naming the source whose authorize
+   // annotation failed) can reuse this 424 mapping without a separate class.
+   constructor(error: { message: string }) {
       super(error.message);
    }
 }
@@ -111,6 +116,18 @@ export class FrozenConfigError extends Error {
    }
 }
 
+/**
+ * A request was denied by a source's `#(authorize)` gate (HTTP 403). Thrown by
+ * the runtime authorize check when no in-scope expression evaluates true for
+ * the supplied givens (including when a referenced given has no value).
+ */
+export class AccessDeniedError extends Error {
+   constructor(message: string) {
+      super(message);
+      this.name = "AccessDeniedError";
+   }
+}
+
 export class MaterializationNotFoundError extends Error {
    constructor(message: string) {
       super(message);

package/src/package_load/package_load_pool.ts

@@ -67,7 +67,6 @@
  *   it resolves.
  */
 import type {
-   Annotation,
    FetchSchemaOptions,
    InfoConnection,
    LookupConnection,
@@ -811,15 +810,11 @@ export class PackageLoadPool {
 
 function buildFetchOptions(options: {
    refreshTimestamp?: number;
-   modelAnnotation?: Annotation;
 }): FetchSchemaOptions {
    const out: FetchSchemaOptions = {};
    if (options.refreshTimestamp !== undefined) {
       out.refreshTimestamp = options.refreshTimestamp;
    }
-   if (options.modelAnnotation !== undefined) {
-      out.modelAnnotation = options.modelAnnotation;
-   }
    return out;
 }
 

package/src/package_load/package_load_worker.ts

@@ -46,7 +46,6 @@
  */
 import {
    contextOverlay,
-   type Annotation,
    type BuildManifestEntry,
    type Connection,
    type FetchSchemaOptions,
@@ -56,16 +55,12 @@ import {
    type ModelDef,
    type ModelMaterializer,
    modelDefToModelInfo,
-   type NamedModelObject,
    type NamedQueryDef,
    type Query,
    Runtime,
    type SQLSourceDef,
    type SQLSourceRequest,
-   type StructDef,
    type TableSourceDef,
-   type TurtleDef,
-   isSourceDef,
 } from "@malloydata/malloy";
 import * as Malloy from "@malloydata/malloy-interfaces";
 import {
@@ -84,7 +79,13 @@ import {
    PACKAGE_MANIFEST_NAME,
 } from "../constants";
 import { HackyDataStylesAccumulator } from "../data_styles";
-import { parseFilters, type FilterDefinition } from "../service/filter";
+import { ModelCompilationError } from "../errors";
+import { validateAuthorizeProbes } from "../service/authorize";
+import { type FilterDefinition } from "../service/filter";
+import {
+   extractQueriesFromModelDef,
+   extractSourcesFromModelDef,
+} from "../service/source_extraction";
 import {
    malloyGivenToApi,
    type MalloyGiven,
@@ -271,15 +272,13 @@ class ProxyConnection {
 
 function serializeFetchOptions(options: FetchSchemaOptions): {
    refreshTimestamp?: number;
-   modelAnnotation?: Annotation;
 } {
-   const out: { refreshTimestamp?: number; modelAnnotation?: Annotation } = {};
+   const out: {
+      refreshTimestamp?: number;
+   } = {};
    if (options.refreshTimestamp !== undefined) {
       out.refreshTimestamp = options.refreshTimestamp;
    }
-   if (options.modelAnnotation !== undefined) {
-      out.modelAnnotation = options.modelAnnotation;
-   }
    return out;
 }
 
@@ -411,6 +410,7 @@ interface ApiSourceWire {
    views?: { name: string; annotations?: string[] }[];
    filters?: unknown[];
    givens?: unknown[];
+   authorize?: string[];
 }
 interface ApiQueryWire {
    name: string;
@@ -472,96 +472,20 @@ function appendLocalSourceInfos(
    }
 }
 
+// Source / query introspection is shared with the in-process path; see
+// service/source_extraction.ts. The worker has no logger, so a filter parse
+// failure is swallowed silently (no `onParseError` callback) — matching the
+// prior worker-local behavior.
 function extractSources(
-   modelPath: string,
    modelDef: ModelDef,
    givens: ApiGivenWire[] | undefined,
 ): { sources: ApiSourceWire[]; filterMap: Map<string, FilterDefinition[]> } {
-   const filterMap = new Map<string, FilterDefinition[]>();
-   const sources: ApiSourceWire[] = Object.values(modelDef.contents)
-      .filter((obj) => isSourceDef(obj))
-      .map((sourceObj) => {
-         const sourceName =
-            (sourceObj as StructDef).as || (sourceObj as StructDef).name;
-         const annotations = (sourceObj as StructDef).annotation?.blockNotes
-            ?.filter((note) => note.at.url.includes(modelPath))
-            .map((note) => note.text);
-
-         const collected: string[][] = [];
-         let cur: Annotation | undefined = (sourceObj as StructDef).annotation;
-         while (cur) {
-            if (cur.blockNotes) {
-               collected.push(cur.blockNotes.map((note) => note.text));
-            }
-            cur = cur.inherits;
-         }
-         const allAnnotations = collected.reverse().flat();
-         let filters: unknown[] | undefined;
-         if (allAnnotations.length > 0) {
-            try {
-               const parsed = parseFilters(allAnnotations);
-               if (parsed.length > 0) {
-                  filterMap.set(sourceName, parsed);
-                  const fields = (sourceObj as StructDef).fields;
-                  filters = parsed.map((f) => {
-                     const field = fields.find(
-                        (fd) => (fd.as || fd.name) === f.dimension,
-                     );
-                     return {
-                        name: f.name,
-                        dimension: f.dimension,
-                        type: f.type,
-                        implicit: f.implicit,
-                        required: f.required,
-                        dimensionType: field?.type as string | undefined,
-                     };
-                  });
-               }
-            } catch {
-               /* parse errors are warnings; matches in-process */
-            }
-         }
-
-         const views = (sourceObj as StructDef).fields
-            .filter((f) => f.type === "turtle")
-            .filter((turtle) =>
-               (turtle as TurtleDef).pipeline
-                  .map((stage) => stage.type)
-                  .every((type) => type === "reduce"),
-            )
-            .map((turtle) => ({
-               name: turtle.as || turtle.name,
-               annotations: turtle?.annotation?.blockNotes
-                  ?.filter((note) => note.at.url.includes(modelPath))
-                  .map((note) => note.text),
-            }));
-
-         return {
-            name: sourceName,
-            annotations,
-            views,
-            filters,
-            givens,
-         } as ApiSourceWire;
-      });
-
-   return { sources, filterMap };
+   const { sources, filterMap } = extractSourcesFromModelDef(modelDef, givens);
+   return { sources: sources as unknown as ApiSourceWire[], filterMap };
 }
 
-function extractQueries(modelPath: string, modelDef: ModelDef): ApiQueryWire[] {
-   const isNamedQuery = (obj: NamedModelObject): obj is NamedQueryDef =>
-      obj.type === "query";
-   return Object.values(modelDef.contents)
-      .filter(isNamedQuery)
-      .map((q) => ({
-         name: q.as || q.name,
-         sourceName: typeof q.structRef === "string" ? q.structRef : undefined,
-         annotations: q?.annotation?.blockNotes
-            ?.filter((note: { at: { url: string } }) =>
-               note.at.url.includes(modelPath),
-            )
-            .map((note: { text: string }) => note.text),
-      }));
+function extractQueries(modelDef: ModelDef): ApiQueryWire[] {
+   return extractQueriesFromModelDef(modelDef) as ApiQueryWire[];
 }
 
 function buildRuntimeForModel(
@@ -621,8 +545,12 @@ async function compileMalloyModel(
    );
    appendLocalSourceInfos(modelDef, sourceInfos, importedNames);
 
-   const { sources, filterMap } = extractSources(modelPath, modelDef, givens);
-   const queries = extractQueries(modelPath, modelDef);
+   const { sources, filterMap } = extractSources(modelDef, givens);
+   const queries = extractQueries(modelDef);
+   // Validate #(authorize) at compile time (shared with Model.create). Throws
+   // on an unknown given / source-field reference; compileOneModel's catch
+   // turns it into this model's compilationError.
+   await validateAuthorizeProbes(mm, sources);
 
    return {
       modelPath,
@@ -798,10 +726,12 @@ async function compileNotebookModel(
          collected.importedNames,
       );
       finalSourceInfos = collected.sourceInfos;
-      const extracted = extractSources(modelPath, finalModelDef, finalGivens);
+      const extracted = extractSources(finalModelDef, finalGivens);
       finalSources = extracted.sources;
       finalFilterMap = extracted.filterMap;
-      finalQueries = extractQueries(modelPath, finalModelDef);
+      finalQueries = extractQueries(finalModelDef);
+      // Validate #(authorize) at compile time (shared with Model.create).
+      await validateAuthorizeProbes(mm, finalSources);
    }
 
    return {
@@ -899,6 +829,18 @@ function serializeError(error: unknown): SerializedError {
          isCompilationError: true,
       };
    }
+   // ModelCompilationError (e.g. an invalid #(authorize) annotation caught by
+   // validateAuthorizeProbes) carries no Malloy `problems`, but it must keep its
+   // compilation-error classification across the worker boundary so the main
+   // thread re-wraps it as a 424, not a generic 500.
+   if (error instanceof ModelCompilationError) {
+      return {
+         name: error.name,
+         message: error.message,
+         stack: error.stack,
+         isCompilationError: true,
+      };
+   }
    if (error instanceof Error) {
       return {
          name: error.name,

package/src/package_load/protocol.ts

@@ -64,11 +64,7 @@
  * cheaper than `JSON.stringify` for the multi-MB modelDef payloads.
  */
 
-import type {
-   Annotation,
-   SQLSourceDef,
-   TableSourceDef,
-} from "@malloydata/malloy";
+import type { SQLSourceDef, TableSourceDef } from "@malloydata/malloy";
 
 // ──────────────────────────────────────────────────────────────────────
 // Direction: main ──▶ worker (load-package job)
@@ -236,7 +232,6 @@ export interface SchemaForTablesRequest {
    tables: Record<string, string>;
    options: {
       refreshTimestamp?: number;
-      modelAnnotation?: Annotation;
    };
 }
 
@@ -256,7 +251,6 @@ export interface SchemaForSqlRequest {
    sentence: unknown;
    options: {
       refreshTimestamp?: number;
-      modelAnnotation?: Annotation;
    };
 }
 

package/src/service/annotations.spec.ts

@@ -0,0 +1,118 @@
+import { describe, expect, it } from "bun:test";
+import type { ModelDef } from "@malloydata/malloy";
+import {
+   annotationTexts,
+   isReservedRoute,
+   modelAnnotations,
+} from "./annotations";
+
+// Minimal `ModelDef` carrying only what `modelAnnotations` reads: `modelID`
+// and the `modelAnnotations` registry (modelID → { ownNotes, inheritsFrom }).
+const makeModelDef = (
+   modelID: string,
+   registry: Record<
+      string,
+      {
+         ownNotes: { notes?: string[]; blockNotes?: string[] };
+         inheritsFrom: string[];
+      }
+   >,
+): ModelDef =>
+   ({
+      modelID,
+      modelAnnotations: Object.fromEntries(
+         Object.entries(registry).map(([id, { ownNotes, inheritsFrom }]) => [
+            id,
+            {
+               ownNotes: {
+                  notes: ownNotes.notes?.map((text) => ({ text, at: {} })),
+                  blockNotes: ownNotes.blockNotes?.map((text) => ({
+                     text,
+                     at: {},
+                  })),
+               },
+               inheritsFrom,
+            },
+         ]),
+      ),
+   }) as unknown as ModelDef;
+
+const noteTexts = (def: ModelDef): string[] =>
+   (modelAnnotations(def).notes ?? []).map((note) => note.text);
+
+describe("isReservedRoute", () => {
+   it("treats the empty route (MOTLY / render config) as reserved", () => {
+      expect(isReservedRoute("")).toBe(true);
+   });
+
+   it("treats Malloy's punctuation sigils as reserved", () => {
+      // Form 2 reserves the entire punct-only namespace, not just the
+      // currently-claimed sigils (`!` `@` `"` `:`).
+      for (const route of ["!", "@", '"', ":", "%", "-", "::"]) {
+         expect(isReservedRoute(route)).toBe(true);
+      }
+   });
+
+   it("treats bracketed app routes as not reserved", () => {
+      for (const route of [
+         "doc",
+         "label",
+         "filter",
+         "bar-chart",
+         "https://example.com/ns",
+         "123",
+         "══SECURITY══",
+      ]) {
+         expect(isReservedRoute(route)).toBe(false);
+      }
+   });
+});
+
+describe("modelAnnotations", () => {
+   it("returns the local model's own `##` notes", () => {
+      const def = makeModelDef("local", {
+         local: { ownNotes: { notes: ["## local"] }, inheritsFrom: [] },
+      });
+      expect(noteTexts(def)).toEqual(["## local"]);
+   });
+
+   it("falls back to an imported model's notes when the local model has no own `##` (authorize must not fail open)", () => {
+      // Regression: a `##(authorize)` declared in an imported file must still
+      // flow into the importing file's file-level gate even when the importing
+      // file declares no `##` of its own. An earlier fold kept an empty local
+      // link at the top, so `.notes` returned [] and the gate failed open.
+      const def = makeModelDef("local", {
+         local: { ownNotes: {}, inheritsFrom: ["base"] },
+         base: {
+            ownNotes: { notes: ['## (authorize) request.user == "admin"'] },
+            inheritsFrom: [],
+         },
+      });
+      expect(noteTexts(def)).toEqual([
+         '## (authorize) request.user == "admin"',
+      ]);
+   });
+
+   it("surfaces the local model's notes at the top when both local and import have `##`", () => {
+      const def = makeModelDef("local", {
+         local: { ownNotes: { notes: ["## local"] }, inheritsFrom: ["base"] },
+         base: { ownNotes: { notes: ["## base"] }, inheritsFrom: [] },
+      });
+      // `.notes` is the top (local); `.texts()` flattens the whole lineage
+      // ancestral-first.
+      expect(noteTexts(def)).toEqual(["## local"]);
+      expect(annotationTexts(modelAnnotations(def))).toEqual([
+         "## base",
+         "## local",
+      ]);
+   });
+
+   it("is cycle-safe and returns an empty bundle for a model with no annotations", () => {
+      const def = makeModelDef("local", {
+         local: { ownNotes: {}, inheritsFrom: ["base"] },
+         base: { ownNotes: {}, inheritsFrom: ["local"] }, // back-edge cycle
+      });
+      expect(modelAnnotations(def)).toEqual({});
+      expect(noteTexts(def)).toEqual([]);
+   });
+});

package/src/service/annotations.ts

@@ -0,0 +1,91 @@
+import { Annotations, type ModelDef } from "@malloydata/malloy";
+
+/**
+ * The raw IR annotation bundle. Workaround: `@malloydata/malloy` exports the
+ * `Annotations` view but not the underlying `AnnotationsDef` type, so we
+ * recover it from the view's constructor parameter. Replace with a direct
+ * `import type { AnnotationsDef }` once malloy exports it.
+ */
+export type AnnotationsDef = NonNullable<
+   ConstructorParameters<typeof Annotations>[0]
+>;
+
+/**
+ * True if a route belongs to Malloy's own reserved namespace rather than an
+ * application. Malloy claims the empty route (`''` — MOTLY tags / render
+ * config) and the whole punctuation-sigil namespace (`!` `@` `"` `:`, and
+ * any punct-only route — Form 2 reserves all of them). Everything else is a
+ * bracketed app route (`#(doc)`, `#<label>`, …).
+ *
+ * TODO: this route classification belongs in `@malloydata/malloy` core,
+ * beside `parsePrefix` — otherwise every consumer reinvents it. Remove when
+ * core exports an equivalent.
+ */
+export function isReservedRoute(route: string): boolean {
+   return route === "" || !/[\p{L}\p{N}]/u.test(route);
+}
+
+/**
+ * The model (`##`) annotation bundle for one model, folded across its
+ * import/extend lineage.
+ *
+ * Workaround: malloy 0.0.405 moved model annotations off `ModelDef.annotation`
+ * and onto `ModelDef.modelAnnotations` (a `modelID → {ownNotes, inheritsFrom}`
+ * registry), folded by the `getModelAnnotations` helper — which malloy does
+ * NOT export from its public barrel. We replicate it (matching
+ * `@malloydata/malloy/dist/model/annotation_utils.js`): a post-order DFS over
+ * `inheritsFrom` (cycle-safe, each model emitted once at its most-ancestral
+ * slot) yields ancestral-first / local-last order, folded into an
+ * `AnnotationsDef` whose `inherits` chain runs most-ancestral-deepest / local
+ * at the top.
+ *
+ * A model that contributes no `##` of its own adds NO link to the chain (we
+ * skip empty `ownNotes`), so `.notes` returns the nearest ancestor that
+ * actually has notes — not an empty local node. This matters because `.notes`
+ * feeds file-level `##(authorize)` enforcement: an imported model's
+ * `##(authorize)` must still flow into an importing file that declares no `##`
+ * of its own. We also copy only `notes`/`blockNotes` rather than spreading
+ * `ownNotes`, whose own `inherits` would otherwise leak in. Replace with a
+ * direct `import { getModelAnnotations }` once malloy exports it.
+ */
+export function modelAnnotations(modelDef: ModelDef): AnnotationsDef {
+   const registry = modelDef.modelAnnotations ?? {};
+   const visited = new Set<string>();
+   const order: string[] = [];
+   const visit = (id: string): void => {
+      if (visited.has(id)) return;
+      visited.add(id);
+      const entry = registry[id];
+      if (!entry) return;
+      for (const dep of entry.inheritsFrom) visit(dep);
+      order.push(id); // post-order: ancestors precede the model itself
+   };
+   visit(modelDef.modelID);
+
+   // Fold most-ancestral → local so the local model lands at the top of the
+   // resulting `inherits` chain. Models with no own notes add no link.
+   let folded: AnnotationsDef | undefined;
+   for (const id of order) {
+      const own = registry[id].ownNotes;
+      if (!own.notes?.length && !own.blockNotes?.length) continue;
+      folded = {
+         notes: own.notes,
+         blockNotes: own.blockNotes,
+         inherits: folded,
+      };
+   }
+   return folded ?? {};
+}
+
+/**
+ * Every annotation text on an entity — its own `notes` and `blockNotes`
+ * plus everything inherited from its ancestors. All of an entity's
+ * annotations apply; none are dropped by source location. Returns
+ * `undefined`, not `[]`, when empty, to match the optional API shape.
+ */
+export function annotationTexts(
+   annote: AnnotationsDef | undefined,
+): string[] | undefined {
+   const texts = new Annotations(annote).texts();
+   return texts.length > 0 ? texts : undefined;
+}